airship/shipyard
Code Issues Proposed changes

Remove Helm 2 based Armada integrations

Browse Source 
- Tiller status check
- Test cleanup arg removed

Depends-On: https://review.opendev.org/c/airship/armada/+/812047
Signed-off-by: Sean Eagan <seaneagan1@gmail.com>
Change-Id: I77ef3fb8e952ad28132e3476138d34bbb5a6fd3d
This commit is contained in:
Sean Eagan 2021-07-28 09:08:16 -05:00
parent a0da68409a
commit e64f17b91b
13 changed files with 26 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
doc/source/action-commands.rst
View File

@ -253,22 +253,14 @@ in all namespaces. Steps, conceptually:
Using test_site
```````````````
The ``test_site`` action accepts two optional parameters:
The ``test_site`` action accepts one optional parameter:
#. cleanup: A boolean value that instructs Armada to delete test pods after
test execution. Default value is ``false``. Failure to set this value to
``True`` may require manual intervention to re-execute tests, as test pods
will not be deleted.
#. release: The name of a release to test. When provided, tests are only
executed for the specified release.
An example of invoking Helm tests with cleanup enabled::
shipyard create action test_site --param="cleanup=true"
An example of invoking Helm tests for a single release::
shipyard create action test_site --param="release=keystone"
shipyard create action test_site --param="namespace=openstack" --param="release=keystone"
.. _update_labels:

11
src/bin/shipyard_airflow/shipyard_airflow/control/action/action_validators.py
View File

@ -27,8 +27,6 @@ from shipyard_airflow.control.validators.validate_intermediate_commit import \
ValidateIntermediateCommit
from shipyard_airflow.control.validators.validate_target_nodes import \
ValidateTargetNodes
from shipyard_airflow.control.validators.validate_test_cleanup import \
ValidateTestCleanup
from shipyard_airflow.shipyard_const import CustomHeaders
LOG = logging.getLogger(__name__)
@ -118,12 +116,3 @@ def validate_target_nodes(action, **kwargs):
"""
validator = ValidateTargetNodes(action=action)
validator.validate()
def validate_test_cleanup(action, **kwargs):
"""Validates the cleanup parameter
Ensures the cleanup parameter is a boolean value.
"""
validator = ValidateTestCleanup(action=action)
validator.validate()

4
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py
View File

@ -92,9 +92,7 @@ def _action_mappings():
'test_site': {
'dag': 'test_site',
'rbac_policy': policy.ACTION_TEST_SITE,
'validators': [
action_validators.validate_test_cleanup,
]
'validators': []
}
}

45
src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_test_cleanup.py
View File

@ -1,45 +0,0 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
from shipyard_airflow.errors import ApiError
class ValidateTestCleanup:
"""Validate that a valid cleanup value is specified for release testing"""
def __init__(self, action):
self.action = action
def validate(self):
"""Retrieve cleanup parameter and verify it is a boolean value"""
# Retrieve optional parameters
parameters = self.action.get('parameters')
if not parameters:
return
# Verify cleanup param (optional) is a boolean value
cleanup = parameters.get('cleanup')
if not cleanup:
return
elif str.lower(cleanup) in ['true', 'false']:
return
raise ApiError(
title='Invalid cleanup value',
description=(
'Cleanup must be a boolean value.'
),
status=falcon.HTTP_400,
retry=False
)

11
src/bin/shipyard_airflow/shipyard_airflow/dags/armada_deploy_site.py
View File

@ -16,14 +16,11 @@ from airflow.models import DAG
try:
from airflow.operators import ArmadaGetReleasesOperator
from airflow.operators import ArmadaGetStatusOperator
from airflow.operators import ArmadaPostApplyOperator
from config_path import config_path
except ImportError:
from shipyard_airflow.plugins.armada_get_releases import \
ArmadaGetReleasesOperator
from shipyard_airflow.plugins.armada_get_status import \
ArmadaGetStatusOperator
from shipyard_airflow.plugins.armada_post_apply import \
ArmadaPostApplyOperator
from shipyard_airflow.dags.config_path import config_path
@ -37,13 +34,6 @@ def deploy_site_armada(parent_dag_name, child_dag_name, args):
'{}.{}'.format(parent_dag_name, child_dag_name),
default_args=args)
# Get Tiller Status
armada_get_status = ArmadaGetStatusOperator(
task_id='armada_get_status',
shipyard_conf=config_path,
main_dag_name=parent_dag_name,
dag=dag)
# Armada Apply
armada_post_apply = ArmadaPostApplyOperator(
task_id='armada_post_apply',
@ -60,7 +50,6 @@ def deploy_site_armada(parent_dag_name, child_dag_name, args):
dag=dag)
# Define dependencies
armada_post_apply.set_upstream(armada_get_status)
armada_get_releases.set_upstream(armada_post_apply)
return dag

69
src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py
View File

@ -1,69 +0,0 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from airflow.exceptions import AirflowException
from airflow.plugins_manager import AirflowPlugin
try:
from armada_base_operator import ArmadaBaseOperator
except ImportError:
from shipyard_airflow.plugins.armada_base_operator import \
ArmadaBaseOperator
from armada.exceptions import api_exceptions as errors
LOG = logging.getLogger(__name__)
class ArmadaGetStatusOperator(ArmadaBaseOperator):
"""Armada Get Status Operator
This operator will trigger armada to get the current status of
Tiller. Tiller needs to be in a healthy state before any site
deployment/update.
"""
def do_execute(self):
# Retrieve read timeout
timeout = self.dc['armada.get_status_timeout']
# Check State of Tiller
try:
armada_get_status = self.armada_client.get_status(
self.query,
timeout=timeout)
except errors.ClientError as client_error:
raise AirflowException(client_error)
# Tiller State will return boolean value, i.e. True/False
# Raise Exception if Tiller is unhealthy
if armada_get_status['tiller']['state']:
LOG.info("Tiller is in running state")
LOG.info("Tiller version is %s",
armada_get_status['tiller']['version'])
else:
raise AirflowException("Please check Tiller!")
class ArmadaGetStatusOperatorPlugin(AirflowPlugin):
"""Creates ArmadaGetStatusOperator in Airflow."""
name = 'armada_get_status_operator'
operators = [ArmadaGetStatusOperator]

6
src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_test_releases.py
View File

@ -33,11 +33,6 @@ class ArmadaTestReleasesOperator(ArmadaBaseOperator):
specified by the "release" parameter.
"""
def do_execute(self):
# Retrieve cleanup flag from action params
cleanup = self.action_params.get('cleanup')
if cleanup:
self.query['cleanup'] = cleanup
release = self.action_params.get('release')
if release:
# Invoke Helm tests for specified release
@ -59,7 +54,6 @@ class ArmadaTestReleasesOperator(ArmadaBaseOperator):
try:
armada_test_release = self.armada_client.get_test_release(
release=release,
query=self.query,
timeout=None)
except errors.ClientError as client_error:
raise AirflowException(client_error)

40
src/bin/shipyard_airflow/tests/unit/control/test_action_helper.py
View File

@ -184,42 +184,24 @@ def test_get_step():
tasks = yaml.safe_load("""
---
- task_id: armada_get_status
- task_id: armada_post_apply
dag_id: update_software.armada_build
execution_date: 2018-09-07 23:18:04
start_date: 2018-09-07 23:18:55.950298
end_date: 2018-09-07 23:18:58.159597
duration: 2.209299
start_date: 2018-09-07 23:48:25.884615
end_date: 2018-09-07 23:48:50.552757
duration: 24.668142
state: success
try_number: 1
hostname: airflow-worker-0.airflow-worker-discovery.ucp.svc.cluster.local
unixname: airflow
job_id: 11
job_id: 13
pool:
queue: default
priority_weight: 3
operator: ArmadaGetStatusOperator
priority_weight: 2
operator: ArmadaPostApplyOperator
queued_dttm:
pid: 249
max_tries: 0
- task_id: armada_get_status
dag_id: update_software.armada_build
execution_date: 2018-09-07 23:18:04
start_date: 2018-09-07 23:18:55.950298
end_date: 2018-09-07 23:18:58.159597
duration: 2.209299
state: success
try_number: 2
hostname: airflow-worker-1.airflow-worker-discovery.ucp.svc.cluster.local
unixname: airflow
job_id: 12
pool:
queue: default
priority_weight: 3
operator: ArmadaGetStatusOperator
queued_dttm:
pid: 249
max_tries: 0
pid: 329
max_tries: 3
- task_id: armada_post_apply
dag_id: update_software.armada_build
execution_date: 2018-09-07 23:18:04
@ -228,7 +210,7 @@ def test_get_step():
duration: 24.668142
state: success
try_number: 2
hostname: airflow-worker-0.airflow-worker-discovery.ucp.svc.cluster.local
hostname: airflow-worker-1.airflow-worker-discovery.ucp.svc.cluster.local
unixname: airflow
job_id: 13
pool:
@ -281,7 +263,7 @@ def test_get_step():
actions_helper = action_helper.ActionsHelper(action_id=action_id)
# Retrieve step
step_id = 'armada_get_status' # task_id in db
step_id = 'armada_post_apply' # task_id in db
# test backward compatibility with no additional param
step = actions_helper.get_step(step_id)

17
src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py
View File

@ -29,7 +29,6 @@ from shipyard_airflow.control.action.action_validators import (
validate_deployment_action_full,
validate_intermediate_commits,
validate_target_nodes,
validate_test_cleanup
)
from shipyard_airflow.errors import ApiError
from tests.unit.common.deployment_group.node_lookup_stubs import node_lookup
@ -273,22 +272,6 @@ class TestActionValidator:
)
assert apie.value.title == 'Invalid target_nodes parameter'
def test_validate_test_cleanup(self, **args):
"""Test that the validate_test_cleanup validator enforces an optional,
boolean value.
"""
# No cleanup param provided
validate_test_cleanup(self._action(None))
# Valid cleanup params
validate_test_cleanup(self._action({'cleanup': 'True'}))
validate_test_cleanup(self._action({'cleanup': 'false'}))
# Bad cleanup params
with pytest.raises(ApiError):
validate_test_cleanup(self._action({'cleanup': 'string'}))
validate_test_cleanup(self._action({'cleanup': '10000'}))
def test_validate_committed_revision(self, *args):
"""Test the committed revision validator"""
validate_committed_revision(self._action(None))

23
src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py
View File

@ -620,22 +620,21 @@ def test_create_targeted_action_no_committed(basic_val, *args):
@mock.patch('shipyard_airflow.control.action.action_validators'
'.validate_target_nodes',
side_effect=Exception('purposeful'))
@mock.patch('shipyard_airflow.control.action.action_validators'
'.validate_test_cleanup',
side_effect=Exception('purposeful'))
@mock.patch('shipyard_airflow.policy.check_auth')
def test_auth_alignment(auth, *args):
action_resource = _gen_action_resource_stubbed()
for action_name, action_cfg in actions_api._action_mappings().items():
with pytest.raises(Exception) as ex:
action = action_resource.create_action(
action={'name': action_name},
context=context,
allow_intermediate_commits=False)
assert 'purposeful' in str(ex)
assert auth.called_with(action_cfg['rbac_policy'])
assert (action_cfg['rbac_policy'] ==
'workflow_orchestrator:action_{}'.format(action_name))
# Only test if validate returns
if action_cfg['validators']:
with pytest.raises(Exception) as ex:
action = action_resource.create_action(
action={'name': action_name},
context=context,
allow_intermediate_commits=False)
assert 'purposeful' in str(ex)
assert auth.called_with(action_cfg['rbac_policy'])
assert (action_cfg['rbac_policy'] ==
'workflow_orchestrator:action_{}'.format(action_name))
@patch('shipyard_airflow.db.shipyard_db.ShipyardDbAccess.'

4
src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py
View File

@ -30,7 +30,6 @@ from shipyard_airflow.plugins.ucp_base_operator import \
CONF_FILE = os.path.join(os.path.dirname(__file__), 'test.conf')
ACTION_PARAMS = {
'cleanup': True,
'release': 'glance'
}
@ -59,7 +58,6 @@ class TestArmadaTestReleasesOperator:
for release in release_list:
calls.append(mock.call(
release=release,
query=dict(),
timeout=None))
mock_client.get_test_release.assert_has_calls(calls, any_order=True)
@ -76,11 +74,9 @@ class TestArmadaTestReleasesOperator:
op.do_execute()
# Verify Armada client called for single release with action params
cleanup = ACTION_PARAMS['cleanup']
release = ACTION_PARAMS['release']
mock_client.get_test_release.assert_called_once_with(
release=release,
query=dict(cleanup=cleanup),
timeout=None)
# Verify test results logged

1
src/bin/shipyard_client/shipyard_client/cli/help/output.py
View File

@ -57,7 +57,6 @@ relabel_nodes
test_site
Triggers the Helm tests for the site, using parameters to control the
tests:
--param="cleanup=true" to delete the test pods immediately after execution
--param="release=release-name" to target a specific Helm release instead of
all releases (the default if this parameter is not specified).
'''

4
tools/test_release.sh
View File

@ -16,7 +16,6 @@
set -ex
# We will need to pass the name of the helm release to test. It is mandatory.
# cleanup may also be passed. By default, test pods will be cleaned up.
# we can execute the script in the following manner:
#
# $ ./test_release.sh helm_release
@ -28,10 +27,9 @@ fi
# Define Variables
helm_release=$1
cleanup=${2:-true}
# Source environment variables
source set_env
# Execute shipyard action for test_site
bash execute_shipyard_action.sh 'test_site' --param="release=${helm_release}" --param="cleanup=${cleanup}"
bash execute_shipyard_action.sh 'test_site' --param="release=${helm_release}"