diff --git a/vmware_nsxlib/tests/unit/v3/test_client.py b/vmware_nsxlib/tests/unit/v3/test_client.py index 39d53752..02e71329 100644 --- a/vmware_nsxlib/tests/unit/v3/test_client.py +++ b/vmware_nsxlib/tests/unit/v3/test_client.py @@ -305,6 +305,10 @@ class NsxV3RESTClientTestCase(nsxlib_testcase.NsxClientTestCase): utils.set_inject_headers_callback(None) self.assertIsNotNone(self.injected) + def test_http_error_to_exception(self): + exc = client.http_error_to_exception(500, 607) + self.assertEqual(exc, nsxlib_exc.APITransactionAborted) + class NsxV3JSONClientTestCase(nsxlib_testcase.NsxClientTestCase): diff --git a/vmware_nsxlib/tests/unit/v3/test_utils.py b/vmware_nsxlib/tests/unit/v3/test_utils.py index 9dfa378c..d2c7b088 100644 --- a/vmware_nsxlib/tests/unit/v3/test_utils.py +++ b/vmware_nsxlib/tests/unit/v3/test_utils.py @@ -291,6 +291,20 @@ class TestNsxV3Utils(nsxlib_testcase.NsxClientTestCase): self.assertRaises(exceptions.NsxLibInvalidInput, func_to_fail, 99) self.assertEqual(max_retries, total_count['val']) + def test_retry_random_tuple(self): + max_retries = 5 + total_count = {'val': 0} + + @utils.retry_random_upon_exception( + (exceptions.NsxLibInvalidInput, exceptions.APITransactionAborted), + max_attempts=max_retries) + def func_to_fail(x): + total_count['val'] = total_count['val'] + 1 + raise exceptions.NsxLibInvalidInput(error_message='foo') + + self.assertRaises(exceptions.NsxLibInvalidInput, func_to_fail, 99) + self.assertEqual(max_retries, total_count['val']) + @mock.patch.object(utils, '_update_max_nsgroups_criteria_tags') @mock.patch.object(utils, '_update_max_tags') @mock.patch.object(utils, '_update_tag_length') diff --git a/vmware_nsxlib/v3/client.py b/vmware_nsxlib/v3/client.py index 80a520a3..72b559f8 100644 --- a/vmware_nsxlib/v3/client.py +++ b/vmware_nsxlib/v3/client.py @@ -40,7 +40,8 @@ def http_error_to_exception(status_code, error_code): requests.codes.CONFLICT: exceptions.StaleRevision, requests.codes.PRECONDITION_FAILED: exceptions.StaleRevision, requests.codes.INTERNAL_SERVER_ERROR: - {'99': exceptions.ClientCertificateNotTrusted}, + {'99': exceptions.ClientCertificateNotTrusted, + '607': exceptions.APITransactionAborted}, requests.codes.FORBIDDEN: {'98': exceptions.BadXSRFToken}, requests.codes.TOO_MANY_REQUESTS: exceptions.TooManyRequests, @@ -313,15 +314,19 @@ class NSX3Client(JSONRESTClient): related_error_codes=related_error_codes) def _rest_call(self, url, **kwargs): - if self.rate_limit_retry and kwargs.get('with_retries', True): - # If too many requests are handled by the nsx at the same time, - # error "429: Too Many Requests" or "503: Server Unavailable" - # will be returned. + if kwargs.get('with_retries', True): + # Retry on "607: Persistence layer is currently reconfiguring" + retry_codes = [exceptions.APITransactionAborted] + if self.rate_limit_retry: + # If too many requests are handled by the nsx at the same time, + # error "429: Too Many Requests" or "503: Server Unavailable" + # will be returned. + retry_codes.append(exceptions.ServerBusy) + # the client is expected to retry after a random 400-600 milli, # and later exponentially until 5 seconds wait @utils.retry_random_upon_exception( - exceptions.ServerBusy, - max_attempts=self.max_attempts) + tuple(retry_codes), max_attempts=self.max_attempts) def _rest_call_with_retry(self, url, **kwargs): return super(NSX3Client, self)._rest_call(url, **kwargs) return _rest_call_with_retry(self, url, **kwargs) diff --git a/vmware_nsxlib/v3/exceptions.py b/vmware_nsxlib/v3/exceptions.py index ef11dc11..f2beab9e 100644 --- a/vmware_nsxlib/v3/exceptions.py +++ b/vmware_nsxlib/v3/exceptions.py @@ -161,3 +161,7 @@ class NsxPendingDelete(NsxLibException): message = _("An object with the same name is marked for deletion. Either " "use another path or wait for the purge cycle to permanently " "remove the deleted object") + + +class APITransactionAborted(ServerBusy): + message = _("API transaction aborted as MP cluster is reconfiguring.")