airship/shipyard
Code Issues Proposed changes

Promenade validateDesign for Shipyard

Browse Source 
SHIPYARD-342: Shipyard Integration with Promenade to Validate Design

Calls Promenade validateDesign API to validate site design.

Change-Id: Ia763983ed9857d4e5b13cfb11d3654e75e6578a4
This commit is contained in:
One-Fine-Day 2018-02-16 15:23:00 -06:00 committed by Aaron Sheffield
parent 676cfff124
commit aca1b60f22
14 changed files with 244 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
charts/shipyard/values.yaml
View File

@ -373,6 +373,8 @@ conf:
destroy_node_query_interval: 30
destroy_node_task_timeout: 900
cluster_join_check_backoff_time: 120
promenade:
service_type: kubernetesprovisioner
keystone_authtoken:
delay_auth_decision: true
auth_type: password

12
docs/source/_static/shipyard.conf.sample
View File

@ -294,6 +294,18 @@
#named_log_levels = keystoneauth:20,keystonemiddleware:20
[promenade]
#
# From shipyard_api
#
# The service type for the service playing the role of Promenade. The specified
# type is used to perform the service lookup in the Keystone service catalog.
# (string value)
#service_type = kubernetesprovisioner
[requests_config]
#

12
src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample
View File

@ -294,6 +294,18 @@
#named_log_levels = keystoneauth:20,keystonemiddleware:20
[promenade]
#
# From shipyard_api
#
# The service type for the service playing the role of Promenade. The specified
# type is used to perform the service lookup in the Keystone service catalog.
# (string value)
#service_type = kubernetesprovisioner
[requests_config]
#

23
src/bin/shipyard_airflow/shipyard_airflow/conf/config.py
View File

@ -89,7 +89,7 @@ SECTIONS = [
),
ConfigSection(
name='shipyard',
title='Shipyard connection info',
title='Shipyard Connection Info',
options=[
cfg.StrOpt(
'service_type',
@ -104,7 +104,7 @@ SECTIONS = [
),
ConfigSection(
name='deckhand',
title='Deckhand connection info',
title='Deckhand Connection Info',
options=[
cfg.StrOpt(
'service_type',
@ -119,7 +119,7 @@ SECTIONS = [
),
ConfigSection(
name='armada',
title='Armada connection info',
title='Armada Connection Info',
options=[
cfg.StrOpt(
'service_type',
@ -134,7 +134,7 @@ SECTIONS = [
),
ConfigSection(
name='drydock',
title='Drydock connection info',
title='Drydock Connection Info',
options=[
cfg.StrOpt(
'service_type',
@ -147,6 +147,21 @@ SECTIONS = [
),
]
),
ConfigSection(
name='promenade',
title='Promenade Connection Info',
options=[
cfg.StrOpt(
'service_type',
default='kubernetesprovisioner',
help=(
'The service type for the service playing the role '
'of Promenade. The specified type is used to perform '
'the service lookup in the Keystone service catalog.'
)
),
]
),
ConfigSection(
name='requests_config',
title='Requests Configuration',

4
src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py
View File

@ -732,6 +732,10 @@ def _get_validation_endpoints():
'name': 'Armada',
'url': val_ep.format(get_endpoint(Endpoints.ARMADA))
},
{
'name': 'Promenade',
'url': val_ep.format(get_endpoint(Endpoints.PROMENADE))
}
]

2
src/bin/shipyard_airflow/shipyard_airflow/control/service_endpoints.py
View File

@ -39,6 +39,7 @@ class Endpoints(enum.Enum):
DRYDOCK = 'drydock'
ARMADA = 'armada'
DECKHAND = 'deckhand'
PROMENADE = 'promenade'
def _get_service_type(endpoint):
@ -57,6 +58,7 @@ def _get_service_type(endpoint):
Endpoints.DRYDOCK: CONF.drydock.service_type,
Endpoints.ARMADA: CONF.armada.service_type,
Endpoints.DECKHAND: CONF.deckhand.service_type,
Endpoints.PROMENADE: CONF.promenade.service_type
}
return endpoint_values.get(endpoint)
raise AppError(

9
src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py
View File

@ -16,6 +16,7 @@ from airflow.models import DAG
from airflow.operators import ArmadaValidateDesignOperator
from airflow.operators import DeckhandValidateSiteDesignOperator
from airflow.operators import DrydockValidateDesignOperator
from airflow.operators import PromenadeValidateSiteDesignOperator
from config_path import config_path
@ -52,4 +53,12 @@ def validate_site_design(parent_dag_name, child_dag_name, args):
retries=3,
dag=dag)
promenade_validate_docs = PromenadeValidateSiteDesignOperator(
task_id='promenade_validate_site_design',
shipyard_conf=config_path,
main_dag_name=parent_dag_name,
sub_dag_name=child_dag_name,
retries=3,
dag=dag)
return dag

44
src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py
View File

@ -12,14 +12,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
from airflow.utils.decorators import apply_defaults
from airflow.plugins_manager import AirflowPlugin
from airflow.exceptions import AirflowException
from service_endpoint import ucp_service_endpoint
from service_token import shipyard_service_token
from ucp_base_operator import UcpBaseOperator
try:
from service_endpoint import ucp_service_endpoint
except ImportError:
from shipyard_airflow.plugins.service_endpoint import ucp_service_endpoint
try:
from service_token import shipyard_service_token
except ImportError:
from shipyard_airflow.plugins.service_token import shipyard_service_token
try:
from ucp_base_operator import UcpBaseOperator
except ImportError:
from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator
LOG = logging.getLogger(__name__)
@ -35,6 +47,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
@apply_defaults
def __init__(self,
deckhand_design_ref=None,
deckhand_svc_type='deckhand',
promenade_svc_endpoint=None,
promenade_svc_type='kubernetesprovisioner',
redeploy_server=None,
@ -42,6 +56,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
*args, **kwargs):
"""Initialization of PromenadeBaseOperator object.
:param deckhand_design_ref: A URI reference to the design documents
:param deckhand_svc_type: Deckhand Service Type
:param promenade_svc_endpoint: Promenade Service Endpoint
:param promenade_svc_type: Promenade Service Type
:param redeploy_server: Server to be redeployed
@ -55,6 +71,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
pod_selector_pattern=[{'pod_pattern': 'promenade-api',
'container': 'promenade-api'}],
*args, **kwargs)
self.deckhand_design_ref = deckhand_design_ref
self.deckhand_svc_type = deckhand_svc_type
self.promenade_svc_endpoint = promenade_svc_endpoint
self.promenade_svc_type = promenade_svc_type
self.redeploy_server = redeploy_server
@ -85,6 +103,26 @@ class PromenadeBaseOperator(UcpBaseOperator):
LOG.info("Promenade endpoint is %s", self.promenade_svc_endpoint)
# Retrieve Deckhand Endpoint Information
deckhand_svc_endpoint = ucp_service_endpoint(
self, svc_type=self.deckhand_svc_type)
LOG.info("Deckhand endpoint is %s", deckhand_svc_endpoint)
# Form Deckhand Design Reference Path
# This URL will be used to retrieve the Site Design YAMLs
deckhand_path = "deckhand+" + deckhand_svc_endpoint
self.deckhand_design_ref = os.path.join(deckhand_path,
"revisions",
str(self.revision_id),
"rendered-documents")
if self.deckhand_design_ref:
LOG.info("Design YAMLs will be retrieved from %s",
self.deckhand_design_ref)
else:
raise AirflowException("Unable to Retrieve Deckhand Revision "
"%d!" % self.revision_id)
class PromenadeBaseOperatorPlugin(AirflowPlugin):

105
src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py Normal file
View File

@ -0,0 +1,105 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import logging
import os
import requests
from airflow.exceptions import AirflowException
from airflow.plugins_manager import AirflowPlugin
try:
from promenade_base_operator import PromenadeBaseOperator
except ImportError:
from shipyard_airflow.plugins.promenade_base_operator import (
PromenadeBaseOperator
)
LOG = logging.getLogger(__name__)
class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator):
"""
Promenade Validate Site Design Operator
This operator will trigger promenade to invoke the validateDesign
Promenade API
"""
def do_execute(self):
LOG.info("Validating site design...")
# Form Validation Endpoint
validation_endpoint = os.path.join(self.promenade_svc_endpoint,
'validatedesign')
LOG.info("Validation Endpoint is %s", validation_endpoint)
# Define Headers and Payload
headers = {
'Content-Type': 'application/json',
'X-Auth-Token': self.svc_token
}
payload = {
'rel': "design",
'href': self.deckhand_design_ref,
'type': "application/x-yaml"
}
# Requests Promenade to validate site design
LOG.info("Waiting for Promenade to validate site design...")
try:
design_validate_response = requests.post(validation_endpoint,
headers=headers,
data=json.dumps(payload))
except requests.exceptions.RequestException as e:
raise AirflowException(e)
# Convert response to string
validate_site_design = design_validate_response.text
# Print response
LOG.info("Retrieving Promenade validate site design response...")
try:
validate_site_design_dict = json.loads(validate_site_design)
LOG.info(validate_site_design_dict)
except json.JSONDecodeError as e:
raise AirflowException(e)
# Check if site design is valid
status = validate_site_design_dict.get('status',
'unspecified')
if status.lower() == 'success':
LOG.info("Promenade Site Design has been successfully validated")
else:
# Dump logs from Promenade pods
self.get_k8s_logs()
raise AirflowException("Promenade Site Design Validation Failed "
"with status: {}!".format(status))
class PromenadeValidateSiteDesignOperatorPlugin(AirflowPlugin):
"""Creates PromenadeValidateSiteDesginOperator in Airflow."""
name = 'promenade_validate_site_design_operator'
operators = [PromenadeValidateSiteDesignOperator]

5
src/bin/shipyard_airflow/shipyard_airflow/plugins/service_token.py
View File

@ -18,7 +18,10 @@ import time
from airflow.exceptions import AirflowException
from service_session import ucp_keystone_session
try:
from service_session import ucp_keystone_session
except ImportError:
from shipyard_airflow.plugins.service_session import ucp_keystone_session
def shipyard_service_token(func):

17
src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py
View File

@ -20,9 +20,20 @@ from airflow.models import BaseOperator
from airflow.plugins_manager import AirflowPlugin
from airflow.utils.decorators import apply_defaults
from get_k8s_logs import get_pod_logs
from get_k8s_logs import K8sLoggingException
from xcom_puller import XcomPuller
try:
from get_k8s_logs import get_pod_logs
except ImportError:
from shipyard_airflow.plugins.get_k8s_logs import get_pod_logs
try:
from get_k8s_logs import K8sLoggingException
except ImportError:
from shipyard_airflow.plugins.get_k8s_logs import K8sLoggingException
try:
from xcom_puller import XcomPuller
except ImportError:
from shipyard_airflow.plugins.xcom_puller import XcomPuller
LOG = logging.getLogger(__name__)

17
src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py
View File

@ -20,9 +20,20 @@ from airflow.models import BaseOperator
from airflow.plugins_manager import AirflowPlugin
from airflow.utils.decorators import apply_defaults
from service_endpoint import ucp_service_endpoint
from xcom_puller import XcomPuller
from xcom_pusher import XcomPusher
try:
from service_endpoint import ucp_service_endpoint
except ImportError:
from shipyard_airflow.plugins.service_endpoint import ucp_service_endpoint
try:
from xcom_puller import XcomPuller
except ImportError:
from shipyard_airflow.plugins.xcom_puller import XcomPuller
try:
from xcom_pusher import XcomPusher
except ImportError:
from shipyard_airflow.plugins.xcom_pusher import XcomPusher
LOG = logging.getLogger(__name__)

4
src/bin/shipyard_airflow/tests/unit/control/test.conf
View File

@ -29,6 +29,10 @@ project_domain_name = default
project_name = service
user_domain_name = default
username = shipyard
[k8s_logs]
ucp_namespace = ucp
[promenade]
service_type = kubernetesprovisioner
[requests_config]
airflow_log_connect_timeout = 5
airflow_log_read_timeout = 300

2
tools/resources/shipyard.conf
View File

@ -42,6 +42,8 @@ user_domain_name = default
username = shipyard
[k8s_logs]
ucp_namespace = ucp
[promenade]
service_type = kubernetesprovisioner
[requests_config]
airflow_log_connect_timeout = 5
airflow_log_read_timeout = 300