airship/shipyard
Code Issues Proposed changes
shipyard/charts/shipyard/values.yaml
Anselme, Schubert (sa246v) f571611f3c
Enable TLS connection to rabbitmq 
Change-Id: Ia4d65393ad0112ae63433a7f67a8c8706a15b216
Signed-off-by: Anselme, Schubert (sa246v) <sa246v@att.com>
2023-11-20 10:42:30 -05:00

1275 lines
38 KiB
YAML
Raw Blame History

# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file provides defaults for shipyard and airflow
labels:
job:
node_selector_key: ucp-control-plane
node_selector_value: enabled
shipyard:
node_selector_key: ucp-control-plane
node_selector_value: enabled
airflow_webserver_ui:
node_selector_key: ucp-control-plane
node_selector_value: enabled
airflow:
node_selector_key: ucp-control-plane
node_selector_value: enabled
test:
node_selector_key: ucp-control-plane
node_selector_value: enabled
images:
tags:
airflow: quay.io/airshipit/airflow:latest-ubuntu_focal
apache_proxy: docker.io/library/httpd:2.4
shipyard: quay.io/airshipit/shipyard:latest-ubuntu_focal
dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
shipyard_db_init: docker.io/postgres:14.8
shipyard_db_auxiliary: docker.io/postgres:14.8
shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_focal
airflow_db_init: docker.io/postgres:14.8
rabbit_init: docker.io/rabbitmq:3.7-management
airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_focal
ks_user: docker.io/openstackhelm/heat:ocata
ks_service: docker.io/openstackhelm/heat:ocata
ks_endpoints: docker.io/openstackhelm/heat:ocata
image_repo_sync: docker.io/docker:17.07.0
pull_policy: "IfNotPresent"
local_registry:
active: false
exclude:
- dep_check
- image_repo_sync
release_group: null
network:
shipyard:
ingress:
public: true
classes:
namespace: "nginx"
cluster: "nginx-cluster"
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
nginx.ingress.kubernetes.io/proxy-body-size: "10m"
nginx.ingress.kubernetes.io/configuration-snippet: |
more_clear_headers "Server";
more_set_headers "X-Content-Type-Options: 'nosniff'";
more_set_headers "X-Frame-Options: 'deny'";
node_port: 31901
enable_node_port: false
airflow:
worker:
name: airflow-worker
port: 8793
enable_node_port: false
airflow_webserver_ui:
ingress:
public: true
classes:
namespace: "nginx"
cluster: "nginx-cluster"
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
nginx.ingress.kubernetes.io/proxy-body-size: "10m"
nginx.ingress.kubernetes.io/configuration-snippet: |
more_clear_headers "Server";
more_set_headers "X-Content-Type-Options: 'nosniff'";
more_set_headers "X-Frame-Options: 'deny'";
node_port: 31902
enable_node_port: false
dependencies:
static:
shipyard_db_init:
jobs:
- airflow-db-init
- airflow-db-sync
services:
- service: postgresql_shipyard_db
endpoint: internal
shipyard_db_auxiliary:
jobs:
- shipyard-db-init
shipyard_db_sync:
jobs:
- shipyard-db-auxiliary
services:
- service: postgresql_shipyard_db
endpoint: internal
airflow_db_init:
services:
- service: postgresql_airflow_db
endpoint: internal
rabbit_init:
services:
- service: oslo_messaging
endpoint: internal
airflow_db_sync:
jobs:
- airflow-db-init
services:
- service: postgresql_airflow_db
endpoint: internal
ks_user:
services:
- service: identity
endpoint: internal
ks_service:
services:
- service: identity
endpoint: internal
ks_endpoints:
jobs:
- shipyard-ks-service
services:
- service: identity
endpoint: internal
shipyard:
jobs:
- shipyard-db-sync
- shipyard-ks-endpoints
- shipyard-ks-user
- shipyard-ks-endpoints
services:
- service: identity
endpoint: internal
- service: postgresql_shipyard_db
endpoint: internal
airflow_server:
jobs:
- airflow-rabbit-init
- airflow-db-sync
services:
- service: postgresql_airflow_db
endpoint: internal
- service: oslo_messaging
endpoint: internal
volume_worker:
class_name: general
size: 5Gi
logrotate:
days_before_deletion: 30
percent_max_log_fs_usage: 80
# typically overriden by environmental
# values, but should include all endpoints
# required by this chart
endpoints:
cluster_domain_suffix: cluster.local
identity:
name: keystone
auth:
shipyard:
region_name: RegionOne
role: admin
project_name: service
project_domain_name: default
user_domain_name: default
username: shipyard
password: password
admin:
region_name: RegionOne
project_name: admin
password: password
username: admin
user_domain_name: default
project_domain_name: default
hosts:
default: keystone
internal: keystone-api
path:
default: /v3
scheme:
default: http
port:
api:
default: 80
internal: 5000
host_fqdn_override:
default: null
shipyard:
name: shipyard
hosts:
default: shipyard-int
public: shipyard-api
port:
api:
default: 9000
public: 80
path:
default: /api/v1.0
scheme:
default: http
host_fqdn_override:
default: null
# NOTE(bryan-strassner): this chart supports TLS for fqdn over-ridden public
# endpoints using the following format:
# public:
# host: null
# tls:
# crt: null
# key: null
airflow_webserver_ui:
name: airflow-webserver-ui
hosts:
default: airflow-int
public: airflow-api
port:
api:
default: 80
public: 80
auth:
admin:
username: airflow
password: password
path:
default: /
scheme:
default: http
host_fqdn_override:
default: null
# NOTE(bryan-strassner): this chart supports TLS for fqdn over-ridden public
# endpoints using the following format:
# public:
# host: null
# tls:
# crt: null
# key: null
airflow_worker:
name: airflow-worker
hosts:
default: airflow-worker
discovery: airflow-worker-discovery
host_fqdn_override:
default: null
path: null
scheme: 'http'
port:
airflow_worker:
default: 8793
postgresql_shipyard_db:
name: postgresql_shipyard_db
auth:
admin:
username: postgres
password: password
user:
username: shipyard
password: password
database: shipyard
hosts:
default: postgresql
path: /shipyard
scheme: postgresql+psycopg2
port:
postgresql:
default: 5432
host_fqdn_override:
default: null
postgresql_airflow_db:
name: postgresql_airflow_db
auth:
admin:
username: postgres
password: password
user:
username: airflow
password: password
database: airflow
hosts:
default: postgresql
path: /airflow
scheme: postgresql+psycopg2
port:
postgresql:
default: 5432
host_fqdn_override:
default: null
postgresql_airflow_celery_db:
name: postgresql_airflow_celery_db
auth:
admin:
username: postgres
password: password
user:
username: airflow
password: password
database: airflow
hosts:
default: postgresql
path: /airflow
scheme: db+postgresql
port:
postgresql:
default: 5432
host_fqdn_override:
default: null
oslo_messaging:
statefulset:
replicas: 1
name: rabbitmq-rabbitmq
auth:
user:
username: airflow
password: password
admin:
username: rabbitmq
password: password
secret:
tls:
internal: rabbitmq-tls-direct
hosts:
default: rabbitmq
host_fqdn_override:
default: null
path: /airflow
scheme: rabbit
port:
amqp:
default: 5672
http:
default: 15672
oslo_cache:
hosts:
default: memcached
host_fqdn_override:
default: null
port:
memcache:
default: 11211
ldap:
hosts:
default: ldap
auth:
admin:
bind: "cn=admin,dc=cluster,dc=local"
password: password
host_fqdn_override:
default: null
path:
default: "/ou=People,dc=cluster,dc=local"
scheme:
default: ldap
port:
ldap:
default: 389
secrets:
identity:
admin: shipyard-keystone-admin
shipyard: shipyard-keystone-user
oslo_messaging:
admin: airflow-rabbitmq-admin
airflow: airflow-rabbitmq-user
postgresql_shipyard_db:
admin: shipyard-db-admin
user: shipyard-db-user
postgresql_airflow_db:
admin: airflow-db-admin
user: airflow-db-user
airflow_webserver_ui:
admin: airflow-admin-creds
tls:
shipyard:
shipyard:
public: shipyard-tls-public
airflow_webserver_ui:
airflow_webserver_ui:
public: airflow-tls-public
conf:
httpd: |
ServerRoot "/usr/local/apache2"
Listen 80
LoadModule mpm_event_module modules/mod_mpm_event.so
LoadModule authn_file_module modules/mod_authn_file.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule ldap_module modules/mod_ldap.so
LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
LoadModule reqtimeout_module modules/mod_reqtimeout.so
LoadModule filter_module modules/mod_filter.so
LoadModule proxy_html_module modules/mod_proxy_html.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule env_module modules/mod_env.so
LoadModule headers_module modules/mod_headers.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_connect_module modules/mod_proxy_connect.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
LoadModule remoteip_module modules/mod_remoteip.so
LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
LoadModule unixd_module modules/mod_unixd.so
LoadModule status_module modules/mod_status.so
LoadModule autoindex_module modules/mod_autoindex.so
<IfModule unixd_module>
User daemon
Group daemon
</IfModule>
<Directory />
AllowOverride none
Require all denied
</Directory>
<Files ".ht*">
Require all denied
</Files>
ErrorLog /dev/stderr
LogLevel warn
<IfModule log_config_module>
LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy
LogFormat "%h %l %u %t \"%r\" %>s %b" common
<IfModule logio_module>
LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
</IfModule>
SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
CustomLog /dev/stdout common
CustomLog /dev/stdout combined
CustomLog /dev/stdout proxy env=forwarded
</IfModule>
<Directory "/usr/local/apache2/cgi-bin">
AllowOverride None
Options None
Require all granted
</Directory>
<IfModule headers_module>
RequestHeader unset Proxy early
</IfModule>
<IfModule proxy_html_module>
Include conf/extra/proxy-html.conf
</IfModule>
<VirtualHost *:80>
RemoteIPHeader X-Original-Forwarded-For
ProxyPreserveHost On
ProxyPass / http://localhost:8080/
ProxyPassReverse / http://localhost:8080/
<Proxy *>
AuthName "Airflow"
AuthType Basic
AuthBasicProvider file ldap
AuthUserFile /usr/local/apache2/conf/.htpasswd
AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
Require valid-user
</Proxy>
</VirtualHost>
uwsgi:
# for more info about these default overrides please read
# https://uwsgi-docs.readthedocs.io/en/latest/ThingsToKnow.html
threads: 1 # Number of threads per worker/process
workers: 16 # maximum number of workers/processes
cheaper_algo: busyness # Use cheaper busyness plugin for dynamic scaling of workers
cheaper: 4 # Minimum number of workers allowed
cheaper_initial: 8 # Workers created at startup
cheaper_step: 4 # How many workers to spawn at a time
cheaper_overload: 1 # Length of a cycle in seconds
cheaper_busyness_multiplier: 60 # Cycles to wait before killing workers
cheaper_busyness_min: 20 # Below this threshold, kill workers
cheaper_busyness_max: 75 # Above this threshold, spawn new workers
cheaper_busyness_backlog_alert: 40 # Spawn emergency workers if more than this many requests are waiting in the queue
cheaper_busyness_backlog_step: 1 # Emergegency workers to create if there are too many requests in the queue
policy:
admin_create: role:admin or role:admin_ucp
admin_read_access: rule:admin_create or role:admin_ucp_viewer
workflow_orchestrator:list_actions: rule:admin_read_access
workflow_orchestrator:create_action: rule:admin_create
workflow_orchestrator:get_action: rule:admin_read_access
workflow_orchestrator:get_action_step: rule:admin_read_access
workflow_orchestrator:get_action_step_logs: rule:admin_read_access
workflow_orchestrator:get_action_validation: rule:admin_read_access
workflow_orchestrator:invoke_action_control: rule:admin_create
workflow_orchestrator:get_configdocs_status: rule:admin_read_access
workflow_orchestrator:create_configdocs: rule:admin_create
workflow_orchestrator:get_configdocs: rule:admin_read_access
workflow_orchestrator:get_configdocs_cleartext: rule:admin_create
workflow_orchestrator:commit_configdocs: rule:admin_create
workflow_orchestrator:get_renderedconfigdocs: rule:admin_read_access
workflow_orchestrator:get_renderedconfigdocs_cleartext: rule:admin_create
workflow_orchestrator:list_workflows: rule:admin_read_access
workflow_orchestrator:get_workflow: rule:admin_read_access
workflow_orchestrator:get_notedetails: rule:admin_read_access
workflow_orchestrator:get_site_statuses: rule:admin_read_access
workflow_orchestrator:action_deploy_site: rule:admin_create
workflow_orchestrator:action_update_site: rule:admin_create
workflow_orchestrator:action_update_software: rule:admin_create
workflow_orchestrator:action_redeploy_server: rule:admin_create
workflow_orchestrator:action_relabel_nodes: rule:admin_create
workflow_orchestrator:action_test_site: rule:admin_create
rabbitmq:
# adding rmq policy to mirror messages from celery queues
policies:
- vhost: "airflow"
name: "ha_celery"
definition:
ha-mode: "all"
ha-sync-mode: "automatic"
priority: 0
apply-to: all
pattern: 'celery|default'
paste:
app:shipyard-api:
paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard
pipeline:main:
pipeline: authtoken shipyard-api
filter:authtoken:
paste.filter_factory: keystonemiddleware.auth_token:filter_factory
shipyard:
base:
web_server: http://localhost:8080/
pool_size: 15
pool_pre_ping: true
pool_timeout: 30
pool_overflow: 10
connection_recycle: -1
profiler: false
shipyard:
service_type: shipyard
deckhand:
service_type: deckhand
armada:
service_type: armada
drydock:
service_type: physicalprovisioner
promenade:
service_type: kubernetesprovisioner
keystone_authtoken:
delay_auth_decision: true
auth_type: password
auth_section: keystone_authtoken
auth_version: v3
memcache_security_strategy: ENCRYPT
requests_config:
airflow_log_connect_timeout: 5
airflow_log_read_timeout: 300
validation_connect_timeout: 20
validation_read_timeout: 300
notes_connect_timeout: 5
notes_read_timeout: 10
deckhand_client_connect_timeout: 20
deckhand_client_read_timeout: 300
drydock_client_connect_timeout: 20
drydock_client_read_timeout: 300
airflow:
worker_endpoint_scheme: 'http'
worker_port: 8793
k8s_logs:
ucp_namespace: 'ucp'
deployment_status_configmap:
name: 'deployment-status'
namespace: 'ucp'
oslo_policy:
policy_file: /etc/shipyard/policy.yaml
# If non-existent rule is used, the request should be denied. The
# deny_all rule is hard coded in the policy.py code to allow no access.
policy_default_rule: deny_all
document_info:
# The name of the deployment version document that Shipyard validates
deployment_version_name: deployment-version
# The schema of the deployment version document that Shipyard validates
deployment_version_schema: pegleg/DeploymentData/v1
# The name of the deployment configuration document that Shipyard expects
# and validates
deployment_configuration_name: deployment-configuration
# The schema of the deployment configuration document that Shipyard
# expects and validates
deployment_configuration_schema: shipyard/DeploymentConfiguration/v1
# The schema of the deployment strategy document that Shipyard expects
# and validates.
deployment_strategy_schema: shipyard/DeploymentStrategy/v1
validations:
# Control the severity of the deployment-version document validation
# that Shipyard performs during create configdocs.
# Possible values are Skip, Info, Warning, and Error
deployment_version_create: Skip
# Control the severity of the deployment-version document validation
# that Shipyard performs during commit configdocs.
# Possible values are Skip, Info, Warning, and Error
deployment_version_commit: Skip
airflow_config_file:
path: /usr/local/airflow/airflow.cfg
airflow_webserver_config_file:
path: /usr/local/airflow/webserver_config.py
airflow_unittests_file:
path: /usr/local/airflow/unittests.cfg
airflow:
# NOTE: Airflow 1.10 introduces a need to declare all config options:
# https://issues.apache.org/jira/browse/AIRFLOW-3099
core:
# core.airflow_home is not used in 1.10.3 and later.
# Envrionment variable AIRFLOW_HOME is used instead.
#airflow_home: /usr/local/airflow
dags_folder: /usr/local/airflow/dags
sensitive_var_conn_names:
- sql_alchemy_conn
- broker_url
- result_backend
- fernet_key
lazy_discover_providers: "True"
lazy_load_plugins: "True"
hostname_callable: "socket:getfqdn"
default_timezone: "utc"
executor: "CeleryExecutor"
# sql_alchemy_conn is extracted from endpoints by the configmap template
sql_engine_encoding: "utf-8"
sql_alchemy_pool_enabled: "True"
sql_alchemy_pool_size: 5
sql_alchemy_pool_recycle: 1800
sql_alchemy_reconnect_timeout: 30
sql_alchemy_schema: ""
parallelism: 32
dag_concurrency: 8
dags_are_paused_at_creation: "False"
# In 1.10.4, non_pooled_task_slot_count and non_pooled_backfill_task_slot_count are
# removed in favor of default_pool, which is initialized with 128 slots by default.
#non_pooled_task_slot_count: 128
max_active_runs_per_dag: 8
load_examples: "False"
plugins_folder: /usr/local/airflow/plugins
fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs=
donot_pickle: "False"
dagbag_import_timeout: 30
# BashTaskRunner has been renamed to StandardTaskRunner from 1.10.3
task_runner: "StandardTaskRunner"
default_impersonation: ""
security: ""
secure_mode: "True"
unit_test_mode: "False"
task_log_reader: "task"
enable_xcom_pickling: "False"
killed_task_cleanup_time: 60
dag_run_conf_overrides_params: "False"
worker_precheck: "False"
cli:
api_client: airflow.api.client.local_client
# if endpoint_url is not set, it is extracted from endpoints by the
# configmap template
endpoint_url: http://localhost/
api:
auth_backend: airflow.api.auth.backend.default
enable_experimental_api: "True"
lineage:
# Shipyard is not using this
backend: ""
logging:
# See image-bundled log_config.py.
# Adds console logging of task/step logs.
# logging_config_class: log_config.LOGGING_CONFIG
logging_config_class: log_config.DEFAULT_LOGGING_CONFIG
# NOTE: Airflow 1.10 introduces extra newline characters between log
# records. Version 1.10.1 should resolve this issue
# https://issues.apache.org/jira/browse/AIRFLOW-1917
#
# NOTE: The log format ends up repeated for each log record that we log
# in our custom operators, once for the logging_mixin class of
# Airflow itself, and once again for the message we want to log.
# E.g.:
# 2018-09-21 19:38:48,950 INFO logging_mixin(95) write - 2018-09-21 19:38:48,950 INFO deployment_configuration_operator(135) get_doc - Deckhand Client acquired
#
# NOTE: Updated from default to match Shipyard logging as much as
# possible without more aggressive techniques
#
log_format: "%%(asctime)s %%(levelname)-8s %%(filename)s:%%(lineno)3d:%%(funcName)s %%(module)s %%(message)s"
log_filename_template: "{{ ti.dag_id }}/{{ ti.task_id }}/{{ execution_date.strftime('%%Y-%%m-%%dT%%H:%%M:%%S') }}/{{ try_number }}.log"
log_processor_filename_template: "{{ filename }}.log"
dag_processor_manager_log_location: /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log
logging_level: "DEBUG"
fab_logging_level: "WARNING"
celery_logging_level: "DEBUG"
base_log_folder: /usr/local/airflow/logs
remote_logging: "False"
remote_log_conn_id: ""
remote_base_log_folder: ""
encrypt_s3_logs: "False"
atlas:
# Shipyard is not using this
sasl_enabled: "False"
host: ""
port: 21000
username: ""
password: ""
operators:
default_owner: "airflow"
default_cpus: 1
default_ram: 512
default_disk: 512
default_gpus: 0
hive:
# Shipyard is not using this
default_hive_mapred_queue: ""
webserver:
# if base_url is not set, is extracted from endpoints by the configmap
# template
base_url: http://localhost/
# set web_server_host to 0.0.0.0 to bind to all interfaces. By default
# only bind to loopback
web_server_host: 127.0.0.1
web_server_port: 8080
web_server_ssl_cert: ""
web_server_ssl_key: ""
web_server_master_timeout: 120
web_server_worker_timeout: 120
worker_refresh_batch_size: 1
worker_refresh_interval: 120
secret_key: "{SECRET_KEY}"
workers: 4
worker_class: "sync"
access_logfile: "-"
error_logfile: "-"
expose_config: "False"
authenticate: "False"
filter_by_owner: "False"
owner_mode: "user"
dag_default_view: "tree"
dag_orientation: "LR"
demo_mode: "False"
log_fetch_timeout_sec: 20
hide_paused_dags_by_default: "False"
page_size: 100
rbac: "False"
navbar_color: "#007A87"
default_dag_run_display_number: 25
enable_proxy_fix: "False"
email:
# Shipyard is not using this
email_backend: airflow.utils.send_email_smtp
smtp:
# Shipyard is not using this
smtp_host: "localhost"
smtp_starttls: "True"
smtp_ssl: "False"
smtp_user: "airflow"
smtp_password: "airflow"
smtp_port: 25
smtp_mail_from: airflow@airflow.local
celery:
celery_app_name: airflow.executors.celery_executor
# The concurrency that will be used when starting workers with the
# "airflow worker" command. This defines the number of task instances
# that a worker will take, so size up your workers based on the resources
# on your worker box and the nature of your tasks
worker_concurrency: 4
worker_log_server_port: 8793
# broker_url is extracted from endpoints by the configmap template
# result_backend is extracted from endpoints by the configmap template
flower_host: 0.0.0.0
flower_url_prefix: ""
flower_port: 5555
flower_basic_auth: ""
operation_timeout: 30.0
broker_connection_retry_on_startup: "True"
broker_connection_retry: "True"
broker_connection_max_retries: 0
broker_connection_timeout: 30.0
default_queue: "default"
# How many processes CeleryExecutor uses to sync task state.
# 0 means to use max(1, number of cores - 1) processes.
# set to 1 for low-volume use
sync_parallelism: 1
celery_config_options: airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
# TODO: Enable this for security
ssl_active: "False"
ssl_key: ""
ssl_cert: ""
ssl_cacert: ""
celery_broker_transport_options:
visibility_timeout: 21600
dask:
# Shipyard is not using this
cluster_adresss: "127.0.0.1:8786"
tls_ca: ""
tls_cert: ""
tls_key: ""
scheduler:
# Task instances listen for external kill signal (when you clear tasks
# from the CLI or the UI), this defines the frequency at which they
# should listen (in seconds).
job_heartbeat_sec: 10
# The scheduler constantly tries to trigger new tasks (look at the
# scheduler section in the docs for more information). This defines
# how often the scheduler should run (in seconds).
scheduler_heartbeat_sec: 10
# Check for pending dag runs no more than every 10 seconds
min_file_process_interval: 10
dag_dir_list_interval: 300
# Stats for the scheduler are minimally useful - every 5 mins is enough
print_stats_interval: 300
child_process_log_directory: /usr/local/airflow/logs/scheduler
scheduler_zombie_task_threshold: 300
catchup_by_default: "True"
max_tis_per_query: 512
# Shipyard's use of Airflow is low volume. 1 Thread is probably enough.
# deprecated in 1.10.14. Replaced with "parsing_processes"
# max_threads: 1
parsing_processes: 1
authenticate: "False"
# Turn off scheduler use of cron intervals by setting this to False.
# DAGs submitted manually in the web UI or with trigger_dag will still run.
use_job_schedule: "False"
metrics:
statsd_on: "False"
statsd_host: "localhost"
statsd_port: 8125
statsd_prefix: "airflow"
ldap:
# Shipyard is not using this
uri: ""
user_filter: "objectClass=*"
user_name_attr: "uid"
group_member_attr: "memberOf"
superuser_filter: ""
data_profiler_filter: ""
bind_user: "cn=Manager,dc=example,dc=com"
bind_password: "insecure"
basedn: "dc=example,dc=com"
cacert: "/etc/ca/ldap_ca.crt"
search_scope: "LEVEL"
mesos:
# Shipyard is not using this
master: ""
framework_name: ""
task_cpu: ""
task_memory: ""
checkpoint: ""
authenticate: ""
kerberos:
# Shipyard is not using this
ccache: ""
principal: ""
reinit_frequency: ""
kinit_path: ""
keytab: ""
github_enterprise:
# Shipyard is not using this
api_rev: v3
admin:
hide_sensitive_variable_fields: "True"
elasticsearch:
# Shipyard is not using this
elasticsearch_host: ""
elasticsearch_log_id_template: ""
elasticsearch_end_of_log_mark: ""
kubernetes:
# Shipyard is not using this (maybe future for spawning own workers)
worker_container_repository: ""
worker_container_tag: ""
delete_worker_pods: "True"
namespace: "default"
airflow_configmap: ""
dags_volume_subpath: ""
dags_volume_claim: ""
logs_volume_subpath: ""
logs_volume_claim: ""
git_repo: ""
git_branch: ""
git_user: ""
git_password: ""
git_subpath: ""
git_sync_container_repository: ""
git_sync_container_tag: ""
git_sync_init_container_name: ""
worker_service_account_name: ""
image_pull_secrets: ""
gcp_service_account_keys: ""
in_cluster: ""
kubernetes_secrets:
#Shipyard is not using this
# End of Airflow config options
pod:
mandatory_access_control:
type: apparmor
shipyard-api:
init: runtime/default
shipyard-api: runtime/default
airflow-web: runtime/default
airflow-worker:
init: runtime/default
worker-perms: runtime/default
airflow-worker: runtime/default
airflow-scheduler: runtime/default
airflow-logrotate: runtime/default
airflow-scheduler:
init: runtime/default
airflow-scheduler: runtime/default
airflow-webserver-ui:
init: runtime/default
apache-proxy: runtime/default
airflow-webserver-ui: runtime/default
shipyard-db-auxiliary:
init: runtime/default
shipyard-db-auxiliary: runtime/default
shipyard-db-init:
init: runtime/default
shipyard-db-init: runtime/default
shipyard-db-sync:
init: runtime/default
shipyard-db-sync: runtime/default
airflow-db-init:
init: runtime/default
airflow-db-init: runtime/default
airflow-db-sync:
init: runtime/default
airflow-db-sync: runtime/default
shipyard-api-test:
shipyard-api-test: runtime/default
security_context:
shipyard:
pod:
runAsUser: 1000
container:
shipyard_api:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
airflow_web:
readOnlyRootFilesystem: false
allowPrivilegeEscalation: false
airflow_scheduler:
pod:
runAsUser: 1000
container:
airflow_scheduler:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
airflow_webserver_ui:
container:
apache_proxy:
runAsUser: 0
readOnlyRootFilesystem: false
airflow_webserver_ui:
runAsUser: 1000
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
airflow_worker:
pod:
runAsUser: 1000
container:
worker_perms:
runAsUser: 0
readOnlyRootFilesystem: true
airflow_scheduler:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
airflow_worker:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
airflow_logrotate:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
db_auxiliary:
pod:
runAsUser: 1000
container:
shipyard_db_auxiliary:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
db_init:
pod:
runAsUser: 1000
container:
shipyard_db_init:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
airflow_db_init:
readOnlyRootFilesystem: false
allowPrivilegeEscalation: false
db_sync:
pod:
runAsUser: 1000
container:
shipyard_db_sync:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
airflow_db_sync:
readOnlyRootFilesystem: false
allowPrivilegeEscalation: false
api_test:
pod:
runAsUser: 1000
container:
shipyard_api_test:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
mounts:
airflow_scheduler:
# TODO: This is only used if the standalone scheduler is enabled.
airflow_scheduler:
init_container: null
airflow_worker:
init_container: null
airflow_worker:
airflow_scheduler:
airflow_webserver_ui:
airflow_webserver_ui:
init_container: null
shipyard:
init_container: null
shipyard:
shipyard_db_init:
init_container: null
shipyard_db_init:
shipyard_db_auxiliary:
init_container: null
shipyard_db_auxiliary:
shipyard_db_sync:
init_container: null
shipyard_db_sync:
affinity:
anti:
type:
default: preferredDuringSchedulingIgnoredDuringExecution
topologyKey:
default: kubernetes.io/hostname
replicas:
shipyard:
api: 2
airflow:
worker: 2
scheduler: 2
airflow_webserver_ui:
airflow_webserver_ui: 1
probes:
airflow_worker:
airflow_worker:
readiness:
enabled: true
params:
periodSeconds: 15
timeoutSeconds: 10
airflow_logrotate:
readiness:
enabled: true
params:
initialDelaySeconds: 15
periodSeconds: 15
timeoutSeconds: 10
lifecycle:
upgrades:
deployments:
revision_history: 3
pod_replacement_strategy: RollingUpdate
rolling_update:
max_unavailable: 1
max_surge: 3
termination_grace_period:
airflow:
timeout: 30
shipyard:
timeout: 30
resources:
enabled: false
airflow:
logrotate:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
scheduler:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
web:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
worker:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
apache_proxy:
limits:
memory: "1024Mi"
cpu: "2000m"
requests:
memory: "128Mi"
cpu: "100m"
airflow_webserver_ui:
airflow_webserver_ui:
limits:
memory: "128Mi"
cpu: "100m"
shipyard_api:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
jobs:
rabbit_init:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
airflow_db_init:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
airflow_db_sync:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
ks_endpoints:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
ks_service:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
ks_user:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
shipyard_db_init:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
shipyard_db_auxiliary:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
shipyard_db_sync:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
test:
shipyard:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
network_policy:
airflow:
ingress:
- {}
egress:
- {}
shipyard:
ingress:
- {}
egress:
- {}
manifests:
configmap_shipyard_bin: true
configmap_shipyard_etc: true
configmap_airflow_bin: true
configmap_airflow_etc: true
configmap_airflow_usr: true
# TODO: Set this to false only if a new deployment, or if the worker pod is
# running the scheduler
deployment_airflow_scheduler: true
deployment_airflow_webserver_ui: true
deployment_shipyard: true
statefulset_airflow_worker: true
ingress_shipyard_api: true
ingress_airflow_webserver_ui: true
job_shipyard_db_init: true
job_shipyard_db_auxiliary: true
job_shipyard_db_sync: true
job_rabbit_init: true
job_airflow_db_init: true
job_airflow_db_sync: true
job_ks_endpoints: true
job_ks_service: true
job_ks_user: true
secret_apache_etc: true
secret_airflow_webserver_ui: true
secret_airflow_db: true
secret_shipyard_db: true
secret_ingress_tls: true
secret_keystone: true
secret_rabbitmq: true
service_shipyard: true
service_shipyard_ingress: true
service_airflow_webserver_ui: true
service_airflow_webserver_ui_ingress: true
service_airflow_worker: true
service_discovery_airflow_worker: true
test_shipyard_api: true
network_policy: false
View Git Blame Copy Permalink