diff --git a/defaults/main.yml b/defaults/main.yml index 5178fa8..f80cfa1 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,5 +1,6 @@ --- # Copyright 2016 Internet Solutions (Pty) Ltd +# Copyright 2017 IBM Corp # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -54,10 +55,15 @@ trove_service_publicurl: "{{ trove_service_publicuri_proto }}://{{ external_lb_v trove_service_internalurl: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ trove_service_port }}/v1.0/%(tenant_id)s" trove_service_adminurl: "{{ trove_service_adminuri_proto }}://{{ internal_lb_vip_address }}:{{ trove_service_port }}/v1.0/%(tenant_id)s" trove_auth_url: "{{ keystone_service_internalurl }}" + +# By default assume the trove service network is an internal one +# It would be good to use the catalog service to lookup these URLs based on the endpoint type +# but currently the Trove code doesn't work properly when endpoint types are specified trove_nova_compute_url: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ nova_service_port }}/v2.1" -trove_cinder_url: "{{ trove_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ cinder_service_port }}/v1" -trove_swift_url: "{{ trove_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ swift_proxy_port }}/v1/AUTH_" -trove_neutron_url: "{{ trove_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ neutron_service_port }}/" +trove_cinder_url: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ cinder_service_port }}/v1" +trove_swift_url: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ swift_proxy_port }}/v1/AUTH_" +trove_neutron_url: "{{ trove_service_internaluri_proto }}://{{ internal_lb_vip_address }}:{{ neutron_service_port }}/" + trove_profiler_enabled: false ## Cap the maximum number of threads / workers when a user value is unspecified. @@ -125,6 +131,39 @@ trove_rabbitmq_use_ssl: False trove_rabbitmq_port: 5672 trove_rabbitmq_servers: "{{ rabbitmq_servers }}" +# The trove guest agent in the deployed DB VMs need access to OpenStack services (keystone, swift, etc) +# and also to rabbitmq. The way it gets access (networking) these services may differ. +# By default assume: +# OpenStack services are accesses through public interfaces +# Infrastructure services (rabbitmq) are accessed through the defined provider network. +# +# The value of 'net_name' field of the provider network network to use for infrastructure services +trove_provider_net_name: dbaas-mgmt +trove_provider_network: "{{ provider_networks|map(attribute='network')|selectattr('net_name','defined')|selectattr('net_name', 'equalto', trove_provider_net_name)|list|first }}" +# The value of 'ip_from_q' field +trove_provider_ip_from_q: "{{ trove_provider_network['ip_from_q'] }}" +# The name of the network address pool +trove_container_net_name: "{{ trove_provider_ip_from_q }}_address" +trove_guest_rabbitmq_servers: "{% for host in groups[rabbitmq_host_group] %}{{ hostvars[host]['container_networks'][trove_container_net_name]['address'] }}{% if not loop.last %},{% endif %}{% endfor %}" +trove_guest_rabbitmq_use_ssl: "{{ trove_rabbitmq_use_ssl }}" +# For OpenStack services that have public, admin, and internal access, use the public ones for the guest VMs. +trove_guest_auth_url: "{{ keystone_service_publicurl }}" +trove_guest_swift_url: "{{ trove_service_publicuri_proto }}://{{ external_lb_vip_address }}:{{ swift_proxy_port }}/v1/AUTH_" + +# Trove service network settings. +# These values are used when creating an OpenStack network to be used by Trove. By default the network will +# not be created. +trove_service_net_setup: False +trove_service_net_validate_certs: "False" +trove_service_net_phys_net: dbaas-mgmt +trove_service_net_type: flat +trove_service_net_name: dbaas_service_net +trove_service_subnet_name: dbaas_service_subnet +trove_service_net_subnet_cidr: "192.168.20.0/24" +trove_service_net_dhcp: "True" +trove_service_net_allocation_pool_start: "192.168.20.100" +trove_service_net_allocation_pool_end: "192.168.20.120" + # RPC encryption keys # See the Trove documentation as to the significance of the rpc encryption keys # Trove supplies default values but we enforce they not be left to their default values @@ -158,6 +197,8 @@ trove_requires_pip_packages: - python-keystoneclient - virtualenv - virtualenv-tools + - shade + - futures trove_pip_packages: - MySQL-python @@ -167,6 +208,7 @@ trove_pip_packages: - trove - python-troveclient - python-memcached + - os-client-config # This variable is used by the repo_build process to determine # which host group to check for members of before building the diff --git a/doc/source/configure-trove.rst b/doc/source/configure-trove.rst new file mode 100644 index 0000000..9860fe8 --- /dev/null +++ b/doc/source/configure-trove.rst @@ -0,0 +1,105 @@ +================= +Configuring Trove +================= + +.. note:: + + Care should be taken when deploying Trove in production environments. + Be sure to fully understand the security implications of the deployed + architecture. + +Trove provides DBaaS to an OpenStack deployment. It deploys guest VMs that +provide the desired DB for use by the end consumer. The trove guest VMs need +connectivity back to the trove services via RPC (rabbitmq) and the OpenStack +services. The way these guest VM get access to those services could be via +internal networking (in the case of rabbitmq) or via public interfaces (in the +case of OpenStack services). For the example configuration, we'll designate +a provider network as the network for trove to provision on each guest VM. The +guest can then connect to rabbitmq via this network and to the OpenStack +services externally. Optionally, the guest VMs could use the internal network +to access OpenStack services, but that would require more containers being +bound to this network. + +The deployment configuration outlined below may not be appropriate for +production environments. Review this very carefully with your own security +requirements. + +Setup a neutron network for use by trove +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Trove needs connectivity between the control plane and the DB guest VMs. For +this purpose a provider network should be created which bridges the trove +containers (if the control plane is installed in a container) or hosts with +VMs. In a general case, neutron networking can be a simple flat network. +An example entry into ``openstack_user_config.yml`` is shown below: + +.. code-block:: yaml + + - network: + container_bridge: "br-dbaas" + container_type: "veth" + container_interface: "eth14" + host_bind_override: "eth14" + ip_from_q: "dbaas" + type: "flat" + net_name: "dbaas-mgmt" + group_binds: + - neutron_linuxbridge_agent + - rabbitmq + +Make sure to modify the other entries in this file as well. + +The ``net_name`` will be the physical network that is specified when creating +the neutron network. The default value of ``dbaas-mgmt`` is also used to +lookup the addresses of the rabbitmq container. If the default is not used then +some variables in ``defaults\main.yml`` will need to be overwritten. + +By default this role will not create the neutron network automaticaly. However, +the default values can be changed to create the neutron network. See the +``trove_service_net_*`` variable in ``defaults\main.yml``. By customizing the +``trove_service_net_*`` variables and having this role create the neutron +network a full deployment of the OpenStack and DBaaS can proceed +without interruption or intervention. + +The following is an example how to set up a provider network in neutron +manually, if so desired: + +.. code-block:: bash + + neutron net-create dbaas_service_net --shared \ + --provider:network_type flat \ + --provider:physical_network dbaas-mgmt + + neutron subnet-create dbaas_service_net 172.19.0.0/22 --name dbaas_service_subnet + --ip-version=4 \ + --allocation-pool start=172.19.1.100,end=172.19.1.200 \ + --enable-dhcp \ + --dns-nameservers list=true 8.8.4.4 8.8.8.8 + +Special attention needs to be applied to the ``--allocation-pool`` to not have +ips which overlap with ips assigned to hosts or containers (see the ``used_ips`` +variable in ``openstack_user_config.yml``) + +.. note:: + This role needs the neutron network created before it can run properly + since the trove guest agent configuration file contains that information. + + +Building Trove images +~~~~~~~~~~~~~~~~~~~~~ + +When building disk image for the guest VM deployments there are many items +to consider. Listed below are a few: + +#. Security of the VM and the network infrastructure +#. What DBs will be installed +#. What DB services will be supported +#. How will the images be maintained + +Images can be built using the ``diskimage-builder`` tooling. The trove +virtual environment can be tar'd up from the trove containers and deployed to +the images using custom ``diskimage-builder`` elements. + +See the ``trove/integration/scripts/files/elements`` directory contents in +the OpenStack Trove project for ``diskimage-builder`` elements to build trove +disk images. diff --git a/doc/source/index.rst b/doc/source/index.rst index 7b0cccd..f74696e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -7,6 +7,11 @@ This Ansible role installs Trove. To clone or view the source code for this repository, visit the role repository for `os_trove `_. +.. toctree:: + :maxdepth: 2 + + configure-trove.rst + Default variables ~~~~~~~~~~~~~~~~~ diff --git a/tasks/main.yml b/tasks/main.yml index f07d1e7..ffd000a 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -43,6 +43,11 @@ tags: - trove-install +- include: trove_service_network.yml + tags: + - trove-install + - trove-config + - include: trove_post_install.yml tags: - trove-install diff --git a/tasks/trove_service_network.yml b/tasks/trove_service_network.yml new file mode 100644 index 0000000..34bd213 --- /dev/null +++ b/tasks/trove_service_network.yml @@ -0,0 +1,77 @@ +--- +# Copyright 2016,2017 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- block: + - name: Get admin tenant id + keystone: + command: get_tenant + tenant_name: admin + endpoint: "{{ keystone_service_adminurl }}" + login_user: "{{ keystone_admin_user_name }}" + login_password: "{{ keystone_auth_admin_password }}" + login_project_name: "{{ keystone_admin_tenant_name }}" + + - name: Store admin tenant id + set_fact: + keystone_admin_tenant_id: "{{ keystone_facts.id }}" + + - name: Create trove service network + os_network: + cloud: default + validate_certs: "{{ trove_service_net_validate_certs }}" + state: present + name: "{{ trove_service_net_name }}" + provider_physical_network: "{{ trove_service_net_phys_net }}" + provider_network_type: "{{ trove_service_net_type }}" + project: "{{ keystone_admin_tenant_id }}" + wait: yes + register: trove_network + run_once: true + + - name: Create trove service subnet + os_subnet: + cloud: default + validate_certs: "{{ trove_service_net_validate_certs }}" + state: present + network_name: "{{ trove_service_net_name }}" + name: "{{ trove_service_subnet_name }}" + allocation_pool_start: "{{ trove_service_net_allocation_pool_start | default(omit) }}" + allocation_pool_end: "{{ trove_service_net_allocation_pool_end | default(omit) }}" + cidr: "{{ trove_service_net_subnet_cidr }}" + enable_dhcp: "{{ trove_service_net_dhcp }}" + project: "{{ keystone_admin_tenant_id }}" + wait: yes + register: subnet_create + run_once: true + # Block end + when: trove_service_net_setup + +# At this point we need to get the neutron network for trove to use. In our role testing +# environment this won't succeed. So we'll bypass it when in developer mode otherwise, +# this task will fail if trove_service_net_setup is false or else if the +# network isn't pre-created by the user. +- name: Get trove service net id + os_networks_facts: + cloud: default + validate_certs: "{{ trove_service_net_validate_certs }}" + wait: yes + name: "{{ trove_service_net_name }}" + run_once: true + when: not trove_developer_mode | bool + +- name: Save trove service net id + set_fact: + trove_service_net_id: "{% if openstack_networks is defined %}{{ openstack_networks[0].id }}{% else %}UNKNOWN{% endif %}" + run_once: true diff --git a/templates/include_db.j2 b/templates/include_db.j2 new file mode 100644 index 0000000..f6c6bba --- /dev/null +++ b/templates/include_db.j2 @@ -0,0 +1,41 @@ +[mysql] +root_on_create = False +tcp_ports = 3306 +volume_support = True +device_path = /dev/vdb + +# Users to ignore for user create/list/delete operations +ignore_users = os_admin, root +ignore_dbs = mysql, information_schema, performance_schema + +[redis] +tcp_ports = 6379 +#redis uses local storage +volume_support = True +# default device_path = None + +[cassandra] +tcp_ports = 7000, 7001, 9042, 9160 +volume_support = True +device_path = /dev/vdb + +[couchbase] +tcp_ports = 8091, 8092, 4369, 11209-11211, 21100-21199 +volume_support = True +device_path = /dev/vdb + +[mongodb] +tcp_ports = 2500, 27017 +volume_support = True +device_path = /dev/vdb +num_config_servers_per_cluster = 1 +num_query_routers_per_cluster = 1 + +[vertica] +tcp_ports = 5433, 5434, 22, 5444, 5450, 4803 +udp_ports = 5433, 4803, 4804, 6453 +volume_support = True +device_path = /dev/vdb +cluster_support = True +cluster_member_count = 3 +api_strategy = trove.common.strategies.cluster.experimental.vertica.api.VerticaAPIStrategy diff --git a/templates/trove-guestagent.conf.j2 b/templates/trove-guestagent.conf.j2 index 8400872..8127f6a 100644 --- a/templates/trove-guestagent.conf.j2 +++ b/templates/trove-guestagent.conf.j2 @@ -2,13 +2,13 @@ [DEFAULT] rpc_backend = {{ trove_rpc_backend }} control_exchange = {{ trove_control_exchange }} -trove_auth_url = {{ trove_auth_url }} +trove_auth_url = {{ trove_guest_auth_url }} nova_proxy_admin_user = {{ trove_service_user_name }} nova_proxy_admin_pass = {{ trove_service_password }} nova_proxy_admin_tenant_name = {{ trove_service_project_name }} {% if trove_swift_enabled is defined %} -swift_url = {{ trove_swift_url }} +swift_url = {{ trove_guest_swift_url }} # Region name of this node. Default value is None. os_region_name = {{ trove_service_region }} # Service type to use when searching catalog. @@ -24,8 +24,8 @@ datastore_registry_ext = mysql:trove.guestagent.datastore.mysql.manager.Manager, root_grant = ALL root_grant_option = True -# For storage to Swift, use the following as defaults: {% if trove_storage_strategy is defined %} +# For storage to Swift, use the following as defaults: storage_strategy = SwiftStorage storage_namespace = trove.common.strategies.storage.swift @@ -53,21 +53,23 @@ log_file = trove-guestagent.log #trace_sqlalchemy = True [oslo_messaging_rabbit] +{% if trove_guest_rabbitmq_servers.split(',') | length > 1 %} +# RabbitMQ HA cluster host:port pairs. (list value) +# Deprecated group/name - [DEFAULT]/rabbit_hosts +rabbit_hosts={% for host in trove_guest_rabbitmq_servers.split(',') %}{{ host }}:{{ trove_rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %} +{% else %} # The RabbitMQ broker address where a single node is used. (string value) # Deprecated group/name - [DEFAULT]/rabbit_host -rabbit_host = {{ trove_rabbitmq_servers }} +rabbit_host = {{ trove_guest_rabbitmq_servers }} # The RabbitMQ broker port where a single node is used. (integer value) # Deprecated group/name - [DEFAULT]/rabbit_port -rabbit_port= {{ trove_rabbitmq_port }} - -# RabbitMQ HA cluster host:port pairs. (list value) -# Deprecated group/name - [DEFAULT]/rabbit_hosts -#rabbit_hosts=$rabbit_host:$rabbit_port +rabbit_port = {{ trove_rabbitmq_port }} +{% endif %} # Connect over SSL for RabbitMQ. (boolean value) # Deprecated group/name - [DEFAULT]/rabbit_use_ssl -#rabbit_use_ssl=false +rabbit_use_ssl = {{ trove_guest_rabbitmq_use_ssl }} # The RabbitMQ userid. (string value) # Deprecated group/name - [DEFAULT]/rabbit_userid diff --git a/templates/trove-taskmanager.conf.j2 b/templates/trove-taskmanager.conf.j2 index 8a41973..b108822 100644 --- a/templates/trove-taskmanager.conf.j2 +++ b/templates/trove-taskmanager.conf.j2 @@ -13,25 +13,44 @@ taskmanager_rpc_encr_key = {{ trove_taskmanager_rpc_encr_key }} inst_rpc_key_encr_key = {{ trove_inst_rpc_key_encr_key }} db_api_implementation = trove.db.sqlalchemy.api -trove_auth_url = {{ trove_auth_url }} -nova_compute_url = {{ trove_nova_compute_url }} -cinder_url = {{ trove_cinder_url }} -swift_url = {{ trove_swift_url }} -neutron_url = {{ trove_neutron_url }} -os_region_name = {{ trove_service_region }} -nova_compute_service_type = compute -cinder_service_type = volumev2 -swift_service_type = object-store -heat_service_type = orchestration -neutron_service_type = network trove_volume_support = True block_device_mapping = vdb device_path = /dev/vdb mount_point = /var/lib/mysql volume_time_out=30 server_delete_time_out=480 -use_nova_server_config_drive = False +use_nova_server_config_drive = True taskmanager_manager=trove.taskmanager.manager.Manager + +# Keystone +trove_auth_url = {{ trove_auth_url }} +os_region_name = {{ trove_service_region }} + +# Nova +nova_compute_url = {{ trove_nova_compute_url }} +nova_compute_service_type = compute +nova_proxy_admin_user = {{ trove_service_user_name }} +nova_proxy_admin_pass = {{ trove_service_password }} +nova_proxy_admin_tenant_name = {{ trove_service_project_name }} + +# Cinder +cinder_url = {{ trove_cinder_url }} +cinder_service_type = volumev2 + +#Swift +swift_url = {{ trove_swift_url }} +swift_service_type = object-store + +# Neutron +neutron_url = {{ trove_neutron_url }} +neutron_service_type = network +network_driver = trove.network.neutron.NeutronDriver +default_neutron_networks = {{ trove_service_net_id }} + +# Heat +heat_service_type = orchestration + +# Notifications exists_notification_transformer = trove.extensions.mgmt.instances.models.NovaNotificationTransformer exists_notification_ticks = 30 notification_service_id = mysql:2f3ff068-2bfb-4f70-9a9d-a6bb65bc084b @@ -50,16 +69,6 @@ dns_instance_entry_factory = trove.dns.designate.driver.DesignateInstanceEntryFa dns_endpoint_url = http://127.0.0.1/v1/ dns_service_type = dns -# Neutron -network_driver = trove.network.nova.NovaNetwork -default_neutron_networks = - -# Nova -nova_compute_url = {{ trove_nova_compute_url }} -nova_proxy_admin_user = {{ trove_service_user_name }} -nova_proxy_admin_pass = {{ trove_service_password }} -nova_proxy_admin_tenant_name = {{ trove_service_project_name }} - # Trove Security Groups for Instances trove_security_groups_support = True trove_security_group_rule_cidr = 0.0.0.0/0 @@ -82,11 +91,9 @@ template_path = /etc/trove/templates/ pydev_debug = disabled # ================= Guestagent related ======================== -#guest_config = /etc/trove/trove-guestagent.conf -# Use 'guest_info = /etc/guest_info' for pre-Kilo compatibility -#guest_info = guest_info.conf -# Use 'injected_config_location = /etc/trove' for pre-Kilo compatibility -#injected_config_location = /etc/trove/conf.d +guest_config = /etc/trove/trove-guestagent.conf +guest_info = guest_info.conf +injected_config_location = /etc/trove/conf.d #cloudinit_location = /etc/trove/cloudinit [database] @@ -101,35 +108,4 @@ enabled = {{ trove_profiler_enabled }} [oslo_messaging_rabbit] rabbit_use_ssl={{ trove_rabbitmq_use_ssl }} -[mysql] -tcp_ports = 3306 -volume_support = True -device_path = /dev/vdb - -[redis] -tcp_ports = 6379 -# redis uses local storage -volume_support = False -# default device_path = None - -[cassandra] -tcp_ports = 7000, 7001, 9042, 9160 -volume_support = True -device_path = /dev/vdb - -[couchbase] -tcp_ports = 8091, 8092, 4369, 11209-11211, 21100-21199 -volume_support = True -device_path = /dev/vdb - -[mongodb] -volume_support = True -device_path = /dev/vdb - -[vertica] -tcp_ports = 5433, 5434, 22, 5444, 5450, 4803 -udp_ports = 5433, 4803, 4804, 6453 -volume_support = True -device_path = /dev/vdb -mount_point = /var/lib/vertica -taskmanager_strategy = trove.common.strategies.cluster.experimental.vertica.taskmanager.VerticaTaskManagerStrategy +{% include 'include_db.j2' %} diff --git a/templates/trove.conf.j2 b/templates/trove.conf.j2 index 172b926..7449d3c 100644 --- a/templates/trove.conf.j2 +++ b/templates/trove.conf.j2 @@ -57,8 +57,13 @@ nova_proxy_admin_pass = {{ trove_service_password }} nova_proxy_admin_tenant_name = {{ trove_service_project_name }} # Neutron -network_driver = trove.network.nova.NovaNetwork -default_neutron_networks = +network_driver = trove.network.neutron.NeutronDriver +default_neutron_networks = {{ trove_service_net_id }} +neutron_url = {{ trove_neutron_url }} + +# Swift +swift_url = {{ trove_swift_url }} +swift_service_type = object-store # Taskmanager queue name taskmanager_queue = taskmanager @@ -106,53 +111,12 @@ enabled = {{ trove_profiler_enabled }} #trace_sqlalchemy = True [ssl] - #cert_file = /path/to/server.crt #key_file = /path/to/server.key #optional: #ca_file = /path/to/ca_file [oslo_messaging_rabbit] -rabbit_use_ssl={{ trove_rabbitmq_use_ssl }} +rabbit_use_ssl = {{ trove_rabbitmq_use_ssl }} -[mysql] -root_on_create = False -tcp_ports = 3306 -volume_support = True -device_path = /dev/vdb - -# Users to ignore for user create/list/delete operations -ignore_users = os_admin, root -ignore_dbs = mysql, information_schema, performance_schema - -[redis] -tcp_ports = 6379 -#redis uses local storage -volume_support = False -# default device_path = None - -[cassandra] -tcp_ports = 7000, 7001, 9042, 9160 -volume_support = True -device_path = /dev/vdb - -[couchbase] -tcp_ports = 8091, 8092, 4369, 11209-11211, 21100-21199 -volume_support = True -device_path = /dev/vdb - -[mongodb] -tcp_ports = 2500, 27017 -volume_support = True -device_path = /dev/vdb -num_config_servers_per_cluster = 1 -num_query_routers_per_cluster = 1 - -[vertica] -tcp_ports = 5433, 5434, 22, 5444, 5450, 4803 -udp_ports = 5433, 4803, 4804, 6453 -volume_support = True -device_path = /dev/vdb -cluster_support = True -cluster_member_count = 3 -api_strategy = trove.common.strategies.cluster.experimental.vertica.api.VerticaAPIStrategy +{% include 'include_db.j2' %} diff --git a/tests/os_trove-overrides.yml b/tests/os_trove-overrides.yml index bcc968d..797c88c 100644 --- a/tests/os_trove-overrides.yml +++ b/tests/os_trove-overrides.yml @@ -27,6 +27,8 @@ trove_rabbitmq_use_ssl: "{{ rabbitmq_use_ssl }}" trove_rabbitmq_password: "secrete" trove_rabbitmq_userid: trove trove_rabbitmq_vhost: /trove +trove_guest_rabbitmq_servers: "{{ trove_rabbitmq_servers }}" +trove_guest_auth_url: "{{ trove_auth_url }}" trove_requirements_git_install_branch: master trove_service_password: "secrete" trove_regular_user_password: "secrete"