x/cloudbase-init
Code Issues Proposed changes

Added retry decorator

Browse Source 
retry_decorator will be used in places where a general
retry pattern is needed (network code mostly).

Change-Id: I64f6fb92b9228c6be854b44c97faa93a446fae2f
This commit is contained in:
Adrian Vladu 2019-08-29 17:35:46 +03:00
parent cd8aa5f2fa
commit ab1396d2aa
2 changed files with 201 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
cloudbaseinit/tests/utils/test_retry_decorator.py Normal file
View File

@ -0,0 +1,115 @@
# Copyright 2019 Cloudbase Solutions SRL
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Unit tests for the utils.retry_decorator module.
"""
import mock
import unittest
import cloudbaseinit.utils.retry_decorator as retry_decorator
class TestException(Exception):
def __init__(self, message=None):
super(TestException, self).__init__(message)
class UtilsTestCase(unittest.TestCase):
def _get_fake_func_with_retry_decorator(self, side_effect,
*args, **kwargs):
func_side_effect = mock.Mock(side_effect=side_effect)
@retry_decorator.retry_decorator(*args, **kwargs)
def fake_func(*_args, **_kwargs):
return func_side_effect(*_args, **_kwargs)
return fake_func, func_side_effect
@mock.patch.object(retry_decorator, 'time')
def test_retry_decorator(self, mock_time):
max_retry_count = 5
max_sleep_time = 2
timeout = max_retry_count + 1
mock_time.time.side_effect = range(timeout)
raised_exc = TestException('fake_exc')
side_effect = [raised_exc] * max_retry_count
side_effect.append(mock.sentinel.ret_val)
(fake_func,
fake_func_side_effect) = self._get_fake_func_with_retry_decorator(
exceptions=TestException,
max_retry_count=max_retry_count,
max_sleep_time=max_sleep_time,
timeout=timeout,
side_effect=side_effect)
ret_val = fake_func(mock.sentinel.arg,
kwarg=mock.sentinel.kwarg)
self.assertEqual(mock.sentinel.ret_val, ret_val)
fake_func_side_effect.assert_has_calls(
[mock.call(mock.sentinel.arg, kwarg=mock.sentinel.kwarg)] *
(max_retry_count + 1))
self.assertEqual(max_retry_count + 1, mock_time.time.call_count)
mock_time.sleep.assert_has_calls(
[mock.call(sleep_time)
for sleep_time in [1, 2, 2, 2, 1]])
@mock.patch.object(retry_decorator, 'time')
def _test_retry_decorator_exceeded(self, mock_time, expected_try_count,
mock_time_side_eff=None,
timeout=None, max_retry_count=None):
raised_exc = TestException('fake_exc')
mock_time.time.side_effect = mock_time_side_eff
(fake_func,
fake_func_side_effect) = self._get_fake_func_with_retry_decorator(
exceptions=TestException,
timeout=timeout,
side_effect=raised_exc)
self.assertRaises(TestException, fake_func)
fake_func_side_effect.assert_has_calls(
[mock.call()] * expected_try_count)
def test_retry_decorator_tries_exceeded(self):
self._test_retry_decorator_exceeded(
max_retry_count=2,
expected_try_count=3)
def test_retry_decorator_time_exceeded(self):
self._test_retry_decorator_exceeded(
mock_time_side_eff=[0, 1, 4],
timeout=3,
expected_try_count=1)
@mock.patch('time.sleep')
def test_retry_decorator_unexpected_exc(self, mock_sleep):
raised_exc = TestException('fake_exc')
fake_func, fake_func_side_effect = (
self._get_fake_func_with_retry_decorator(
exceptions=(IOError, AttributeError),
side_effect=raised_exc))
self.assertRaises(TestException,
fake_func, mock.sentinel.arg,
fake_kwarg=mock.sentinel.kwarg)
self.assertFalse(mock_sleep.called)
fake_func_side_effect.assert_called_once_with(
mock.sentinel.arg, fake_kwarg=mock.sentinel.kwarg)

86
cloudbaseinit/utils/retry_decorator.py Normal file
View File

@ -0,0 +1,86 @@
# Copyright 2019 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from oslo_log import log as logging
from oslo_utils import excutils
from oslo_utils import reflection
LOG = logging.getLogger(__name__)
def retry_decorator(max_retry_count=5, timeout=None, inc_sleep_time=1,
max_sleep_time=1, exceptions=Exception):
"""Retries invoking the decorated method in case of expected exceptions.
:param max_retry_count: The maximum number of retries performed. If 0, no
retry is performed. If None, there will be no limit
on the number of retries.
:param timeout: The maximum time for which we'll retry invoking the method.
If 0 or None, there will be no time limit.
:param inc_sleep_time: The time sleep increment used between retries.
:param max_sleep_time: The maximum time to wait between retries.
:param exceptions: A list of expected exceptions for which retries will be
performed. If None, any exception will be retried.
"""
def wrapper(f):
def inner(*args, **kwargs):
try_count = 0
sleep_time = 0
time_start = time.time()
while True:
try:
return f(*args, **kwargs)
except exceptions as exc:
with excutils.save_and_reraise_exception() as ctxt:
time_elapsed = time.time() - time_start
time_left = (timeout - time_elapsed
if timeout else 'undefined')
tries_left = (max_retry_count - try_count
if max_retry_count is not None
else 'undefined')
should_retry = (
tries_left and
(time_left == 'undefined' or
time_left > 0))
ctxt.reraise = not should_retry
if should_retry:
try_count += 1
func_name = reflection.get_callable_name(f)
sleep_time = min(sleep_time + inc_sleep_time,
max_sleep_time)
if timeout:
sleep_time = min(sleep_time, time_left)
LOG.debug("Got expected exception %(exc)s while "
"calling function %(func_name)s. "
"Retries left: %(retries_left)s. "
"Time left: %(time_left)s. "
"Time elapsed: %(time_elapsed)s "
"Retrying in %(sleep_time)s seconds.",
dict(exc=exc,
func_name=func_name,
retries_left=tries_left,
time_left=time_left,
time_elapsed=time_elapsed,
sleep_time=sleep_time))
time.sleep(sleep_time)
return inner
return wrapper