airship/shipyard
Code Issues Proposed changes

Make Request Timeout Configurable

Browse Source 
As the size of the YAMLs increases, the amount of time needed
to process the request increased as well. Hence there is a need
to make 'timeout' configurable for the deckhand client.

Change-Id: Iab91091cd8b9a900ad0daeac22e435d4e5c9c97d
This commit is contained in:
Anthony Lin 2018-02-06 02:45:32 +00:00
parent 5ca2b349a2
commit 25236ac89b
7 changed files with 99 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
charts/shipyard/values.yaml
View File

@ -334,6 +334,11 @@ conf:
auth_section: keystone_authtoken
auth_version: v3
memcache_security_strategy: ENCRYPT
requests_config:
deckhand_client_connect_timeout: 5
deckhand_client_read_timeout: 300
validation_connect_timeout: 5
validation_read_timeout: 300
airflow:
override:
append:

15
etc/shipyard/shipyard.conf.sample
View File

@ -277,6 +277,21 @@
#log_level = 10
[requests_config]
# Deckhand client connect timeout (in seconds)
#deckhand_client_connect_timeout = 5
# Deckhand client timeout (in seconds) for GET,
# PUT, POST and DELETE request
#deckhand_client_read_timeout = 300
# UCP component validation connect timeout (in seconds)
#validation_connect_timeout = 5
# UCP component validation timeout (in seconds)
#validation_read_timeout = 300
[shipyard]
#

37
shipyard_airflow/conf/config.py
View File

@ -77,7 +77,7 @@ SECTIONS = [
help=(
'The service type for the service playing the role '
'of Shipyard. The specified type is used to perform '
'the service lookup in the Keystone service catalog. '
'the service lookup in the Keystone service catalog.'
)
),
]
@ -92,7 +92,7 @@ SECTIONS = [
help=(
'The service type for the service playing the role '
'of Deckhand. The specified type is used to perform '
'the service lookup in the Keystone service catalog. '
'the service lookup in the Keystone service catalog.'
)
),
]
@ -107,7 +107,7 @@ SECTIONS = [
help=(
'The service type for the service playing the role '
'of Armada. The specified type is used to perform '
'the service lookup in the Keystone service catalog. '
'the service lookup in the Keystone service catalog.'
)
),
]
@ -122,7 +122,7 @@ SECTIONS = [
help=(
'The service type for the service playing the role '
'of Drydock. The specified type is used to perform '
'the service lookup in the Keystone service catalog. '
'the service lookup in the Keystone service catalog.'
)
),
cfg.IntOpt(
@ -182,6 +182,35 @@ SECTIONS = [
),
]
),
ConfigSection(
name='requests_config',
title='Requests Configuration',
options=[
cfg.IntOpt(
'deckhand_client_connect_timeout',
default=5,
help='Deckhand client connect timeout (in seconds)'
),
cfg.IntOpt(
'deckhand_client_read_timeout',
default=300,
help=(
'Deckhand client timeout (in seconds) for GET, '
'PUT, POST and DELETE request'
)
),
cfg.IntOpt(
'validation_connect_timeout',
default=5,
help='UCP component validation connect timeout (in seconds)'
),
cfg.IntOpt(
'validation_read_timeout',
default=300,
help='UCP component validation timeout (in seconds)'
),
]
),
]

12
shipyard_airflow/control/configdocs/configdocs_helper.py
View File

@ -437,13 +437,13 @@ class ConfigdocsHelper(object):
'content-type': 'application/json'
}
# TODO: We will need to make timeout a configurable value as it
# will differ from site to site based on the size of the rendered
# document
# Note that 30 seconds is not sufficient to complete validations.
# Hence we are increaing the default timeout to 60 seconds.
http_resp = requests.post(
url, headers=headers, data=design_reference, timeout=(5, 60))
url,
headers=headers,
data=design_reference,
timeout=(
CONF.requests_config.validation_connect_timeout,
CONF.requests_config.validation_read_timeout))
# 400 response is "valid" failure to validate. > 400 is a problem.
if http_resp.status_code > 400:
http_resp.raise_for_status()

48
shipyard_airflow/control/configdocs/deckhand_client.py
View File

@ -362,11 +362,14 @@ class DeckhandClient(object):
headers['content-type'] = 'application/x-yaml'
DeckhandClient._log_request('PUT', url, params)
response = requests.put(url,
params=params,
headers=headers,
data=document_data,
timeout=(5, 30))
response = requests.put(
url,
params=params,
headers=headers,
data=document_data,
timeout=(
CONF.requests_config.deckhand_client_connect_timeout,
CONF.requests_config.deckhand_client_read_timeout))
return response
except RequestException as rex:
LOG.error(rex)
@ -386,10 +389,13 @@ class DeckhandClient(object):
}
DeckhandClient._log_request('GET', url, params)
response = requests.get(url,
params=params,
headers=headers,
timeout=(5, 30))
response = requests.get(
url,
params=params,
headers=headers,
timeout=(
CONF.requests_config.deckhand_client_connect_timeout,
CONF.requests_config.deckhand_client_read_timeout))
return response
except RequestException as rex:
LOG.error(rex)
@ -411,11 +417,14 @@ class DeckhandClient(object):
headers['content-type'] = 'application/x-yaml'
DeckhandClient._log_request('POST', url, params)
response = requests.post(url,
params=params,
headers=headers,
data=document_data,
timeout=(5, 30))
response = requests.post(
url,
params=params,
headers=headers,
data=document_data,
timeout=(
CONF.requests_config.deckhand_client_connect_timeout,
CONF.requests_config.deckhand_client_read_timeout))
return response
except RequestException as rex:
LOG.error(rex)
@ -434,10 +443,13 @@ class DeckhandClient(object):
}
DeckhandClient._log_request('DELETE', url, params)
response = requests.delete(url,
params=params,
headers=headers,
timeout=(5, 30))
response = requests.delete(
url,
params=params,
headers=headers,
timeout=(
CONF.requests_config.deckhand_client_connect_timeout,
CONF.requests_config.deckhand_client_read_timeout))
return response
except RequestException as rex:
LOG.error(rex)

5
tests/unit/control/test.conf
View File

@ -35,6 +35,11 @@ project_domain_name = default
project_name = service
user_domain_name = default
username = shipyard
[requests_config]
deckhand_client_connect_timeout=5
deckhand_client_read_timeout=300
validation_connect_timeout=5
validation_read_timeout=300
[shipyard]
service_type = shipyard

5
tools/resources/shipyard.conf
View File

@ -37,5 +37,10 @@ project_domain_name = default
project_name = service
user_domain_name = default
username = shipyard
[requests_config]
deckhand_client_connect_timeout=5
deckhand_client_read_timeout=300
validation_connect_timeout=5
validation_read_timeout=300
[shipyard]
service_type = shipyard