airship/shipyard
Code Issues Proposed changes

Add 'allow-intermediate-commits' option for Site Action

Browse Source 
Shipyard Create Action will stop user from executing a site
action when the current committed revision of documents has
other prior commits that have not been used as part of a
site action, e.g. update_site.

We can re-run the action with the 'allow-intermediate-commits'
option if we are aware of these other commits and if they are
intended.

This patch set updates the relevant Shipyard API, API client,
CLI as well as document for this new option.

It also made some small updates to the 'commit configdocs'
documentation.

Change-Id: I94a4cad313a69dd6fbc626ab923c6eae5cf3db7a
This commit is contained in:
Anthony Lin 2018-04-24 10:33:19 +00:00
parent dffb8d3b93
commit d3d3a0ecb7
10 changed files with 198 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
docs/source/API.rst
View File

@ -421,6 +421,17 @@ a DAG invocation), perform any checks to validate the preconditions to run the
DAG, and trigger the invocation of the DAG. The DAG will run asynchronously in
airflow.
Query Parameters
''''''''''''''''
allow-intermediate-commits=true | **false**
By default, false. User will not be able to continue with a site action,
e.g. update_site if the current committed revision of documents has other
prior commits that have not been used as part of a site action. With
allow-intermediate-commits=true, it allows user to override the default
behavior and continue with the site action. This may be the case when the
user is aware of the existence of such commits and/or when such commits are
intended.
Responses
'''''''''
201 Created

44
docs/source/CLI.rst
View File

@ -107,6 +107,7 @@ downstream components.
shipyard commit configdocs
[--force]
[--dryrun]
Example:
shipyard commit configdocs
@ -122,7 +123,42 @@ Sample
::
TBD
$ shipyard commit configdocs
Configuration documents committed.
Status: Validations succeeded
Reason: Validation
- Info: DD1001
Message: Rational Boot Storage: Validation successful.
Source: Drydock
- Info: DD2002
Message: IP Locality Check: Validation successful.
Source: Drydock
- Info: DD2003
Message: MTU Rationality: Validation successful.
Source: Drydock
- Info: DD2004
Message: Network Trunking Rationalty: Validation successful.
Source: Drydock
- Info: DD2005
Message: Duplicated IP Check: Validation successful.
Source: Drydock
- Info: DD3001
Message: Platform Selection: Validation successful.
Source: Drydock
- Info: DD1006
Message: Network Bond Rationality: Validation successful.
Source: Drydock
- Info: DD2002
Message: Storage Partitioning: Validation successful.
Source: Drydock
- Info: DD2003
Message: Storage Sizing: Validation successful.
Source: Drydock
- Info: DD1007
Message: Allowed Network Check: Validation successful.
Source: Drydock
#### Errors: 0, Warnings: 0, Infos: 10, Other: 0 ####
Control commands
----------------
@ -206,6 +242,7 @@ id of the action invoked so that it can be queried subsequently.
shipyard create action
<action_command>
--param=<parameter> (repeatable)
[--allow-intermediate-commits]
Example:
shipyard create action redeploy_server --param="server-name=mcp"
@ -216,6 +253,11 @@ id of the action invoked so that it can be queried subsequently.
\--param=<parameter>
A parameter to be provided to the action being invoked. (repeatable)
\--allow-intermediate-commits
Allows continuation of a site action, e.g. update_site even when the
current committed revision of documents has other prior commits that
have not been used as part of a site action.
Sample
^^^^^^

42
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py
View File

@ -74,15 +74,24 @@ class ActionsResource(BaseResource):
"""
Accept an action into shipyard
"""
# The 'allow-intermediate-commits' query parameter is set to False
# unless explicitly set to True
allow_intermediate_commits = (
req.get_param_as_bool(name='allow-intermediate-commits'))
input_action = self.req_json(req, validate_json_schema=ACTION)
action = self.create_action(action=input_action, context=req.context)
action = self.create_action(
action=input_action,
context=req.context,
allow_intermediate_commits=allow_intermediate_commits)
LOG.info("Id %s generated for action %s", action['id'], action['name'])
# respond with the action and location for checking status
resp.status = falcon.HTTP_201
resp.body = self.to_json(action)
resp.location = '/api/v1.0/actions/{}'.format(action['id'])
def create_action(self, action, context):
def create_action(self, action, context, allow_intermediate_commits=False):
# use uuid assigned for this request as the id of the action.
action['id'] = ulid.ulid()
# the invoking user
@ -99,10 +108,15 @@ class ActionsResource(BaseResource):
dag = SUPPORTED_ACTION_MAPPINGS.get(action['name'])['dag']
action['dag_id'] = dag
# Retrieve last committed design revision
# Set up configdocs_helper
self.configdocs_helper = ConfigdocsHelper(context)
# Retrieve last committed design revision
action['committed_rev_id'] = self.get_committed_design_version()
# Check for intermediate commit
self.check_intermediate_commit_revision(allow_intermediate_commits)
# populate action parameters if they are not set
if 'parameters' not in action:
action['parameters'] = {}
@ -339,3 +353,25 @@ class ActionsResource(BaseResource):
description='No committed version found in Deckhand',
status=falcon.HTTP_404,
retry=False)
def check_intermediate_commit_revision(self,
allow_intermediate_commits=False):
LOG.info("Checking for intermediate committed revision in Deckhand...")
intermediate_commits = (
self.configdocs_helper.check_intermediate_commit())
if intermediate_commits and not allow_intermediate_commits:
raise ApiError(
title='Intermediate Commit Detected',
description=(
'The current committed revision of documents has '
'other prior commits that have not been used as '
'part of a site action, e.g. update_site. If you '
'are aware and these other commits are intended, '
'please rerun this action with the option '
'`allow-intermediate-commits=True`'),
status=falcon.HTTP_409,
retry=False
)

42
src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_helper.py
View File

@ -720,3 +720,45 @@ class ConfigdocsHelper(object):
# reset the revision dict so it regenerates.
self.revision_dict = None
return self._get_buffer_rev_id()
def check_intermediate_commit(self):
# Initialize variable
list_of_committed_rev = []
try:
# Get the list of all revisions present in Deckhand
all_revisions = self.deckhand.get_revision_list()
except NoRevisionsExistError:
# the values of None/None/None/0 are fine
pass
except DeckhandResponseError as drex:
raise AppError(
title='Unable to retrieve revisions',
description=(
'Deckhand has responded unexpectedly: {}:{}'.format(
drex.status_code, drex.response_message)),
status=falcon.HTTP_500,
retry=False)
if all_revisions:
# Get the list of 'committed' revisions
for revision in all_revisions:
if 'committed' in revision['tags']:
list_of_committed_rev.append(revision)
# This is really applicable for scenarios where multiple
# configdocs commits and site actions were performed. Hence
# we should expect at least 2 'committed' revisions to be
# present in deckhand.
#
# We will check the second last most recent committed revision
# to see if a site-action has been executed on it
if len(list_of_committed_rev) > 1:
if (('site-action-success' and 'site-action-failure') not in
list_of_committed_rev[-2]['tags']):
return True
return False

20
src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py
View File

@ -39,6 +39,10 @@ CONF = cfg.CONF
LOG = logging.getLogger(__name__)
def CHECK_INTERMEDIATE_COMMIT(allow_intermediate_commits):
return False
def create_req(ctx, body):
'''creates a falcon request'''
env = testing.create_environ(
@ -261,7 +265,9 @@ def test_on_post(mock_info, mock_create_action, mock_authorize):
mock_authorize.assert_called_once_with(
'workflow_orchestrator:create_action', context)
mock_create_action.assert_called_once_with(
action=json.loads(json_body.decode('utf-8')), context=context)
action=json.loads(json_body.decode('utf-8')),
context=context,
allow_intermediate_commits=None)
mock_info.assert_called_with("Id %s generated for action %s", 'test_id',
'test_name')
assert resp.status == '201 Created'
@ -299,6 +305,8 @@ def test_create_action():
action_resource.insert_action = insert_action_stub
action_resource.audit_control_command_db = audit_control_command_db
action_resource.get_committed_design_version = lambda: DESIGN_VERSION
action_resource.check_intermediate_commit_revision = (
CHECK_INTERMEDIATE_COMMIT)
# with invalid input. fail.
try:
@ -307,7 +315,8 @@ def test_create_action():
'parameters': {
'a': 'aaa'
}},
context=context)
context=context,
allow_intermediate_commits=False)
assert False, 'Should throw an ApiError'
except ApiError:
# expected
@ -320,7 +329,8 @@ def test_create_action():
'parameters': {
'a': 'aaa'
}},
context=context)
context=context,
allow_intermediate_commits=False)
assert action['timestamp']
assert action['id']
assert len(action['id']) == 26
@ -333,7 +343,9 @@ def test_create_action():
# with valid input and no parameters
try:
action = action_resource.create_action(
action={'name': 'deploy_site'}, context=context)
action={'name': 'deploy_site'},
context=context,
allow_intermediate_commits=False)
assert action['timestamp']
assert action['id']
assert len(action['id']) == 26

12
src/bin/shipyard_client/shipyard_client/api_client/shipyard_api_client.py
View File

@ -122,20 +122,26 @@ class ShipyardClient(BaseClient):
)
return self.get_resp(url)
def post_actions(self, name=None, parameters=None):
def post_actions(self, name=None, parameters=None,
allow_intermediate_commits=False):
"""
Creates an action in the system. This will cause some action to start.
:param str name: name of supported action to invoke
:param dict parameters: parameters to use for trigger invocation
:param allow_intermediate_commits: boolean, True|False
:returns: action entity created successfully
:rtype: Response object
"""
query_params = (
{"allow-intermediate-commits": allow_intermediate_commits})
action_data = {"name": name, "parameters": parameters}
url = ApiPaths.POST_GET_ACTIONS.value.format(
self.get_endpoint()
)
return self.post_resp(
url, data=json.dumps(action_data), content_type='application/json')
return self.post_resp(url=url,
query_params=query_params,
data=json.dumps(action_data),
content_type='application/json')
def get_action_detail(self, action_id=None):
"""

15
src/bin/shipyard_client/shipyard_client/cli/create/actions.py
View File

@ -17,19 +17,24 @@ from shipyard_client.cli import format_utils
class CreateAction(CliAction):
"""Action to Create Action"""
def __init__(self, ctx, action_name, param):
def __init__(self, ctx, action_name, param, allow_intermediate_commits):
"""Sets parameters."""
super().__init__(ctx)
self.logger.debug("CreateAction action initialized with action command"
"%s and parameters %s", action_name, param)
self.logger.debug(
"CreateAction action initialized with action command "
"%s, parameters %s and allow-intermediate-commits=%s",
action_name, param, allow_intermediate_commits)
self.action_name = action_name
self.param = param
self.allow_intermediate_commits = allow_intermediate_commits
def invoke(self):
"""Returns the response from API Client"""
self.logger.debug("Calling API Client post_actions.")
return self.get_api_client().post_actions(name=self.action_name,
parameters=self.param)
return self.get_api_client().post_actions(
name=self.action_name,
parameters=self.param,
allow_intermediate_commits=self.allow_intermediate_commits)
# Handle 400, 409 with default error handler for cli.
cli_handled_err_resp_codes = [400, 409]

15
src/bin/shipyard_client/shipyard_client/cli/create/commands.py
View File

@ -39,7 +39,7 @@ DESC_ACTION = """
DESCRIPTION: Invokes the specified workflow through Shipyard. Returns the
id of the action invoked so that it can be queried subsequently. \n
FORMAT: shipyard create action <action command> --param=<parameter>
(repeatable) \n
(repeatable) [--allow-intermediate-commits] \n
EXAMPLE: shipyard create action redeploy_server --param="server-name=mcp"
"""
@ -55,8 +55,14 @@ SHORT_DESC_ACTION = (
'--param',
multiple=True,
help="A parameter to be provided to the action being invoked.(Repeatable)")
@click.option(
'--allow-intermediate-commits',
'allow_intermediate_commits',
flag_value=True,
help="Allow site action to go through even though there are prior commits "
"that have not been used as part of a site action.")
@click.pass_context
def create_action(ctx, action_name, param):
def create_action(ctx, action_name, param, allow_intermediate_commits=False):
check_action_command(ctx, action_name)
if not param and action_name is 'redeploy_server':
@ -65,7 +71,10 @@ def create_action(ctx, action_name, param):
else:
param = check_reformat_parameter(ctx, param)
click.echo(
CreateAction(ctx, action_name, param).invoke_and_return_resp())
CreateAction(ctx,
action_name,
param,
allow_intermediate_commits).invoke_and_return_resp())
DESC_CONFIGDOCS = """

24
src/bin/shipyard_client/tests/unit/cli/create/test_create_actions.py
View File

@ -45,9 +45,11 @@ def test_create_action(*args):
'http://shiptest/actions',
body=resp_body,
status=201)
response = CreateAction(stubs.StubCliContext(),
action_name='deploy_site',
param=None).invoke_and_return_resp()
response = CreateAction(
stubs.StubCliContext(),
action_name='deploy_site',
param=None,
allow_intermediate_commits=False).invoke_and_return_resp()
assert 'Name' in response
assert 'Action' in response
assert 'Lifecycle' in response
@ -64,9 +66,11 @@ def test_create_action_400(*args):
body=stubs.gen_err_resp(message='Error_400',
reason='bad action'),
status=400)
response = CreateAction(stubs.StubCliContext(),
action_name='deploy_dogs',
param=None).invoke_and_return_resp()
response = CreateAction(
stubs.StubCliContext(),
action_name='deploy_dogs',
param=None,
allow_intermediate_commits=False).invoke_and_return_resp()
assert 'Error_400' in response
assert 'bad action' in response
assert 'action/01BTTMFVDKZFRJM80FGD7J1AKN' not in response
@ -81,9 +85,11 @@ def test_create_action_409(*args):
body=stubs.gen_err_resp(message='Error_409',
reason='bad validations'),
status=409)
response = CreateAction(stubs.StubCliContext(),
action_name='deploy_site',
param=None).invoke_and_return_resp()
response = CreateAction(
stubs.StubCliContext(),
action_name='deploy_site',
param=None,
allow_intermediate_commits=False).invoke_and_return_resp()
assert 'Error_409' in response
assert 'bad validations' in response
assert 'action/01BTTMFVDKZFRJM80FGD7J1AKN' not in response

2
src/bin/shipyard_client/tests/unit/cli/create/test_create_commands.py
View File

@ -35,7 +35,7 @@ def test_create_action():
runner.invoke(shipyard,
[auth_vars, 'create', 'action', action_name, param])
mock_method.assert_called_once_with(ANY, action_name,
{'"server-name': 'mcp"'})
{'"server-name': 'mcp"'}, False)
def test_create_action_negative():