ab1396d2aa
retry_decorator will be used in places where a general retry pattern is needed (network code mostly). Change-Id: I64f6fb92b9228c6be854b44c97faa93a446fae2f
87 lines
3.8 KiB
Python
87 lines
3.8 KiB
Python
# Copyright 2019 Cloudbase Solutions Srl
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import time
|
|
|
|
from oslo_log import log as logging
|
|
from oslo_utils import excutils
|
|
from oslo_utils import reflection
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
def retry_decorator(max_retry_count=5, timeout=None, inc_sleep_time=1,
|
|
max_sleep_time=1, exceptions=Exception):
|
|
"""Retries invoking the decorated method in case of expected exceptions.
|
|
|
|
:param max_retry_count: The maximum number of retries performed. If 0, no
|
|
retry is performed. If None, there will be no limit
|
|
on the number of retries.
|
|
:param timeout: The maximum time for which we'll retry invoking the method.
|
|
If 0 or None, there will be no time limit.
|
|
:param inc_sleep_time: The time sleep increment used between retries.
|
|
:param max_sleep_time: The maximum time to wait between retries.
|
|
:param exceptions: A list of expected exceptions for which retries will be
|
|
performed. If None, any exception will be retried.
|
|
"""
|
|
|
|
def wrapper(f):
|
|
def inner(*args, **kwargs):
|
|
try_count = 0
|
|
sleep_time = 0
|
|
time_start = time.time()
|
|
|
|
while True:
|
|
try:
|
|
return f(*args, **kwargs)
|
|
except exceptions as exc:
|
|
with excutils.save_and_reraise_exception() as ctxt:
|
|
time_elapsed = time.time() - time_start
|
|
time_left = (timeout - time_elapsed
|
|
if timeout else 'undefined')
|
|
tries_left = (max_retry_count - try_count
|
|
if max_retry_count is not None
|
|
else 'undefined')
|
|
|
|
should_retry = (
|
|
tries_left and
|
|
(time_left == 'undefined' or
|
|
time_left > 0))
|
|
ctxt.reraise = not should_retry
|
|
|
|
if should_retry:
|
|
try_count += 1
|
|
func_name = reflection.get_callable_name(f)
|
|
|
|
sleep_time = min(sleep_time + inc_sleep_time,
|
|
max_sleep_time)
|
|
if timeout:
|
|
sleep_time = min(sleep_time, time_left)
|
|
|
|
LOG.debug("Got expected exception %(exc)s while "
|
|
"calling function %(func_name)s. "
|
|
"Retries left: %(retries_left)s. "
|
|
"Time left: %(time_left)s. "
|
|
"Time elapsed: %(time_elapsed)s "
|
|
"Retrying in %(sleep_time)s seconds.",
|
|
dict(exc=exc,
|
|
func_name=func_name,
|
|
retries_left=tries_left,
|
|
time_left=time_left,
|
|
time_elapsed=time_elapsed,
|
|
sleep_time=sleep_time))
|
|
time.sleep(sleep_time)
|
|
return inner
|
|
return wrapper
|