Bryan Strassner 38e58cfd30 Add Action API
This change introduces a large section of the API for the next major
version of Shipyard - the action api.  By interfacing with Airflow,
Shipyard will invoke workflows and allow for controlling and querying
status of those workflows. Foundationally, this patchset introduces
a lot of framework code for other apis, including error handling
to a common output format, database interaction for persistence of
action information, and use of oslo_config for configuration support.

Add GET all actions primary code - db connection not yet impl
Update base classes to have more structure
Add POST actions framework
Add GET action by id
Add GET of validations and steps
Add control api
Add unit tests of action api methods
Re-Removed duplicate deps from test reqs
Add routes for API
Removed a lot of code better handled by falcon directly
Cleaned up error flows- handlers and defaults
Refactored existing airflow tests to match standard output format
Updated json validation to be more specific
Added basic start for alembic
Added alembic upgrade at startup
Added table creation definitions
Added base revision for alembic upgrade
Bug fixes - DB queries, airflow comm, logic issues, logging issues
Bug fixes - date formats and alignment of keys between systems
Exclusions to bandit / tox.ini
Resolved merge conflicts with integration of auth
Update to use oslo config and PBR
Update the context middleware to check uuid in a less contentious way
Removed routes and resources for regions endpoint - not used
Add auth policies for action api
Restructure execptions to be consistent class hierarchy and common handler
Add generation of config and policy examples
Update tests to init configs
Update database configs to not use env. vars
Removed examples directory, it was no longer accurate
Addressed/removed several TODOs - left some behind as well
Aligned input to DAGs with action: header
Retrieved all sub-steps for dags
Expanded step information
Refactored auth handling for better logging
rename create_actions policy to create_action
removed some templated file comments in env.py generated by alembic
updated inconsistent exception parameters
updated to use ulid instead of uuid for action ids
added action control audit code per review suggestion
Fixed correlation date betwen dags/actions by more string parsing

Change-Id: I2f9ea5250923f45456aa86826e344fc055bba762
2017-09-22 21:47:13 -05:00

189 lines
6.2 KiB
Python

# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import logging
import uuid
import falcon
import falcon.request as request
import falcon.routing as routing
from shipyard_airflow.control.json_schemas import validate_json
from shipyard_airflow.errors import InvalidFormatError
class BaseResource(object):
def __init__(self):
self.logger = logging.getLogger('shipyard.control')
def on_options(self, req, resp, **kwargs):
"""
Handle options requests
"""
method_map = routing.create_http_method_map(self)
for method in method_map:
if method_map.get(method).__name__ != 'method_not_allowed':
resp.append_header('Allow', method)
resp.status = falcon.HTTP_200
def req_json(self, req, validate_json_schema=None):
"""
Reads and returns the input json message, optionally validates against
a provided jsonschema
:param req: the falcon request object
:param validate_json_schema: the optional jsonschema to use for
validation
"""
has_input = False
if ((req.content_length is not None or req.content_length != 0) and
(req.content_type is not None and
req.content_type.lower() == 'application/json')):
raw_body = req.stream.read(req.content_length or 0)
if raw_body is not None:
has_input = True
self.info(req.context, 'Input message body: %s' % raw_body)
else:
self.info(req.context, 'No message body specified')
if has_input:
# read the json and validate if necessary
try:
raw_body = raw_body.decode('utf-8')
json_body = json.loads(raw_body)
if validate_json_schema:
# rasises an exception if it doesn't validate
validate_json(json_body, validate_json_schema)
return json_body
except json.JSONDecodeError as jex:
self.error(req.context, "Invalid JSON in request: \n%s" %
raw_body)
raise InvalidFormatError(
title='JSON could not be decoded',
description='%s: Invalid JSON in body: %s' %
(req.path, jex)
)
else:
# No body passed as input. Fail validation if it was asekd for
if validate_json_schema is not None:
raise InvalidFormatError(
title='Json body is required',
description='%s: Bad input, no body provided' %
(req.path)
)
else:
return None
def to_json(self, body_dict):
"""
Thin wrapper around json.dumps, providing the default=str config
"""
return json.dumps(body_dict, default=str)
def log_message(self, ctx, level, msg):
"""
Logs a message with context, and extra populated.
"""
extra = {'user': 'N/A', 'req_id': 'N/A', 'external_ctx': 'N/A'}
if ctx is not None:
extra = {
'user': ctx.user,
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
}
self.logger.log(level, msg, extra=extra)
def debug(self, ctx, msg):
"""
Debug logger for resources, incorporating context.
"""
self.log_message(ctx, logging.DEBUG, msg)
def info(self, ctx, msg):
"""
Info logger for resources, incorporating context.
"""
self.log_message(ctx, logging.INFO, msg)
def warn(self, ctx, msg):
"""
Warn logger for resources, incorporating context.
"""
self.log_message(ctx, logging.WARN, msg)
def error(self, ctx, msg):
"""
Error logger for resources, incorporating context.
"""
self.log_message(ctx, logging.ERROR, msg)
class ShipyardRequestContext(object):
"""
Context object for shipyard resource requests
"""
def __init__(self):
self.log_level = 'error'
self.user = None
self.roles = ['anyone']
self.request_id = str(uuid.uuid4())
self.external_marker = None
self.project_id = None
self.user_id = None # User ID (UUID)
self.policy_engine = None
self.user_domain_id = None # Domain owning user
self.project_domain_id = None # Domain owning project
self.is_admin_project = False
self.authenticated = False
def set_log_level(self, level):
if level in ['error', 'info', 'debug']:
self.log_level = level
def set_user(self, user):
self.user = user
def set_project(self, project):
self.project = project
def add_role(self, role):
self.roles.append(role)
def add_roles(self, roles):
self.roles.extend(roles)
def remove_role(self, role):
self.roles = [x for x in self.roles if x != role]
def set_external_marker(self, marker):
self.external_marker = marker
def set_policy_engine(self, engine):
self.policy_engine = engine
def to_policy_view(self):
policy_dict = {}
policy_dict['user_id'] = self.user_id
policy_dict['user_domain_id'] = self.user_domain_id
policy_dict['project_id'] = self.project_id
policy_dict['project_domain_id'] = self.project_domain_id
policy_dict['roles'] = self.roles
policy_dict['is_admin_project'] = self.is_admin_project
return policy_dict
class ShipyardRequest(request.Request):
context_type = ShipyardRequestContext