From ad8bda5f641c88bb74055197aa248d2b16993d16 Mon Sep 17 00:00:00 2001 From: James Gibson Date: Mon, 25 Oct 2021 08:32:04 +0100 Subject: [PATCH] Enable TLS for live migrations Instead of using SSH to live migrate VM's use TLS as this is more secure and SSH migrations are deprecated. https://docs.openstack.org/nova/xena/admin/secure-live-migration-with-qemu-native-tls.html A pre-existing PKI (Public Key Infrastruture) setup is required. TLS live migrations require that all compute hosts can communcate with each other on port 16514 and port range 49152 to 49261. To enable TLS live migrations, both libvirt and QEMU require server and client certificates, the server certicicates is used to verify servers and the client cert is used by servers to authenticate clients. A single cert is created by the pki role, that can be used by both libvirt and QEMU for both client and server auth. The client, server and CA certifcates need to installed in a number of locations on each compute host: * For Libvirt https://libvirt.org/tlscerts.html * For QEMU https://github.com/libvirt/libvirt/blob/master/src/qemu/qemu.conf Depends-On: https://review.opendev.org/c/openstack/ansible-role-pki/+/815007 Depends-On: https://review.opendev.org/c/openstack/ansible-role-pki/+/815849 Depends-On: https://review.opendev.org/c/openstack/ansible-role-pki/+/816857 Change-Id: Iddbe8764bb6d3cd3eaee122b2d5ddc02fa3f7662 --- defaults/main.yml | 108 +++++++++++++++++- handlers/main.yml | 3 + .../notes/tls-migration-3ed93cc04dab5eee.yaml | 13 +++ tasks/main.yml | 36 +++--- tasks/nova_compute.yml | 24 ++-- templates/libvirtd.conf.j2 | 6 + templates/nova.conf.j2 | 5 + templates/qemu.conf.j2 | 46 ++++++++ 8 files changed, 212 insertions(+), 29 deletions(-) create mode 100644 releasenotes/notes/tls-migration-3ed93cc04dab5eee.yaml diff --git a/defaults/main.yml b/defaults/main.yml index 59434c95..4ed393fe 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -388,7 +388,7 @@ nova_api_threads: "{{ [[(ansible_facts['processor_vcpus']//ansible_facts['proces nova_service_in_ldap: "{{ service_ldap_backend_enabled | default(False) }}" ## libvirtd config options -nova_libvirtd_listen_tls: 0 +nova_libvirtd_listen_tls: 1 nova_libvirtd_listen_tcp: 0 nova_libvirtd_auth_tcp: sasl nova_libvirtd_debug_log_filters: "3:remote 4:event 3:json 3:rpc" @@ -534,3 +534,109 @@ nova_pci_passthrough_whitelist: {} # - '{ "name": "card-alias1", "product_id": "XXXX", "vendor_id": "XXXX" }' # - '{ "name": "card-alias2", "product_id": "XXXY", "vendor_id": "XXXY" }' nova_pci_alias: [] + +# Storage location for SSL certificate authority +nova_pki_dir: "{{ openstack_pki_dir }}" + +# Delegated host for operating the certificate authority +nova_pki_setup_host: "{{ openstack_pki_setup_host | default('localhost') }}" + +# Nova server certificate +nova_pki_keys_path: "{{ nova_pki_dir ~ '/certs/private/' }}" +nova_pki_certs_path: "{{ nova_pki_dir ~ '/certs/certs/' }}" +nova_pki_intermediate_cert_name: "{{ openstack_pki_service_intermediate_cert_name }}" +nova_pki_intermediate_chain_path: "{{ nova_pki_dir ~ '/roots/' ~ nova_pki_intermediate_cert_name ~ '/certs/' ~ nova_pki_intermediate_cert_name ~ '-chain.crt' }}" +nova_pki_regen_cert: '' +# Create client and server cert for compute hosts +# This certiticate is used during TLS live migrations +nova_pki_certificates: + - name: "nova_{{ ansible_facts['hostname'] }}" + provider: ownca + cn: "{{ ansible_facts['nodename'] }}" + san: "{{ 'DNS:' ~ ansible_facts['hostname'] ~ ',DNS:' ~ ansible_facts['nodename'] ~ ',IP:' ~ (nova_management_address == 'localhost') | ternary('127.0.0.1', nova_management_address) }}" + signed_by: "{{ nova_pki_intermediate_cert_name }}" + key_usage: + - digitalSignature + - keyAgreement + - keyEncipherment + extended_key_usage: + - clientAuth + - serverAuth + +# libvirt default destination files for SSL certificates +nova_libvirt_ssl_dir: /etc/pki/libvirt +# QEMU default destination files for SSL certificates +nova_qemu_ssl_dir: /etc/pki/qemu + +# Installation details for SSL certificates for TLS live migration +nova_pki_install_certificates: + # Server certificate used by libvirt for live migrations + - src: "{{ nova_user_ssl_cert | default(nova_pki_certs_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '-chain.crt') }}" + dest: "{{ nova_libvirt_ssl_dir }}/servercert.pem" + owner: "root" + group: "root" + mode: "0640" + # Server certificate key used by libvirt for live migrations + - src: "{{ nova_user_ssl_key | default(nova_pki_keys_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '.key.pem') }}" + dest: "{{ nova_libvirt_ssl_dir }}/private/serverkey.pem" + owner: "root" + group: "root" + mode: "0640" + # Client certificate used by libvirt for live migrations + # Defaults to using the server certificate which is signed for both clientAuth and serverAuth + - src: "{{ nova_user_ssl_cert | default(nova_pki_certs_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '-chain.crt') }}" + dest: "{{ nova_libvirt_ssl_dir }}/clientcert.pem" + owner: "root" + group: "root" + mode: "0640" + # Client certificate key used by libvirt for live migrations + - src: "{{ nova_user_ssl_key | default(nova_pki_keys_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '.key.pem') }}" + dest: "{{ nova_libvirt_ssl_dir }}/private/clientkey.pem" + owner: "root" + group: "root" + mode: "0640" + # Server certificate used by QEMU for live migrations + - src: "{{ nova_user_ssl_cert | default(nova_pki_certs_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '-chain.crt') }}" + dest: "{{ nova_qemu_ssl_dir }}/server-cert.pem" + owner: "root" + group: "{{ nova_qemu_user }}" + mode: "0640" + # Server certificate key used by QEMU for live migrations + - src: "{{ nova_user_ssl_key | default(nova_pki_keys_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '.key.pem') }}" + dest: "{{ nova_qemu_ssl_dir }}/server-key.pem" + owner: "root" + group: "{{ nova_qemu_user }}" + mode: "0640" + # Client certificate used by QEMU for live migrations + # Defaults to using the server certificate which is signed for both clientAuth and serverAuth + - src: "{{ nova_user_ssl_cert | default(nova_pki_certs_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '-chain.crt') }}" + dest: "{{ nova_qemu_ssl_dir }}/client-cert.pem" + owner: "root" + group: "{{ nova_qemu_user }}" + mode: "0640" + # Client certificate key used by QEMU for live migrations + - src: "{{ nova_user_ssl_key | default(nova_pki_keys_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '.key.pem') }}" + dest: "{{ nova_qemu_ssl_dir }}/client-key.pem" + owner: "root" + group: "{{ nova_qemu_user }}" + mode: "0640" + # Root CA for libvirt + # libvirt requires that the CA cert file has any intermediate certificates for the server cert, + # so defaults to using the intermediate chain, which contains the intermediate and Root CA + - src: "{{ nova_user_ssl_ca_cert | default(nova_pki_intermediate_chain_path) }}" + dest: "/etc/pki/CA/cacert.pem" + owner: "root" + group: "root" + mode: "0644" + # Root CA for qemu + - src: "{{ nova_user_ssl_ca_cert | default(nova_pki_intermediate_chain_path) }}" + dest: "{{ nova_qemu_ssl_dir }}/ca-cert.pem" + owner: "root" + group: "root" + mode: "0644" + +# Define user-provided SSL certificates in: +# /etc/openstack_deploy/user_variables.yml +#nova_user_ssl_cert: +#nova_user_ssl_key: +#nova_user_ssl_ca_cert: diff --git a/handlers/main.yml b/handlers/main.yml index 60704077..01b9aa61 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -20,6 +20,7 @@ state: "stopped" listen: - Restart libvirt-bin + - "cert installed" - name: Enable sockets when needed service: @@ -36,6 +37,7 @@ condition: "{{ nova_libvirtd_listen_tcp | bool }}" listen: - Restart libvirt-bin + - "cert installed" - name: Start libvirt-bin service: @@ -44,6 +46,7 @@ state: "started" listen: - Restart libvirt-bin + - "cert installed" - name: Stop services service: diff --git a/releasenotes/notes/tls-migration-3ed93cc04dab5eee.yaml b/releasenotes/notes/tls-migration-3ed93cc04dab5eee.yaml new file mode 100644 index 00000000..f0bfe601 --- /dev/null +++ b/releasenotes/notes/tls-migration-3ed93cc04dab5eee.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Nova now defaults to to using the "QEMU-native TLS" feature + for live migrations, rather than the deprecated SSH method. + A pre-existing PKI (Public Key Infrastructure) setup is + required. + + QEMU-native TLS requires all compute hosts to accept TCP connections on + port 16514 and port range 49152 to 49261. + + More information can be found here: + https://docs.openstack.org/nova/latest/admin/secure-live-migration-with-qemu-native-tls.html diff --git a/tasks/main.yml b/tasks/main.yml index e582ba38..49840d0a 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -34,26 +34,6 @@ tags: - always -- name: Fail if TCP and TLS are both enabled - fail: - msg: | - TCP and TLS connectivity are currently enabled for libvirtd. This - combination prevents libvirtd from starting properly since this role - does not generate TLS certificates for libvirtd at this time. - - To enable TCP connectivity without TLS, set the following variables: - - nova_libvirtd_listen_tcp: 1 - nova_libvirtd_listen_tls: 0 - - Please note that this configuration does not encrypt communication with - libvirtd. - when: - - nova_libvirtd_listen_tcp == 1 - - nova_libvirtd_listen_tls == 1 - tags: - - always - - name: Fail if service was deployed using a different installation method fail: msg: "Switching installation methods for OpenStack services is not supported" @@ -139,6 +119,22 @@ tags: - nova-config +# Create certs after libvirt groups have been created but before handlers +- name: Create and install SSL certificates + include_role: + name: pki + tasks_from: main_certs.yml + vars: + pki_setup_host: "{{ nova_pki_setup_host }}" + pki_dir: "{{ nova_pki_dir }}" + pki_create_certificates: "{{ nova_user_ssl_cert is not defined and nova_user_ssl_key is not defined }}" + pki_regen_certificates: "{{ nova_pki_regen_cert }}" + pki_certificates: "{{ nova_pki_certificates }}" + pki_install_certificates: "{{ nova_pki_install_certificates }}" + when: + - nova_libvirtd_listen_tls == 1 + - "'nova_compute' in group_names" + - import_tasks: nova_post_install.yml tags: - nova-config diff --git a/tasks/nova_compute.yml b/tasks/nova_compute.yml index eb9116ba..85ec0143 100644 --- a/tasks/nova_compute.yml +++ b/tasks/nova_compute.yml @@ -21,15 +21,23 @@ tags: - always -- import_tasks: nova_compute_key_populate.yml - tags: - - nova-config - - nova-key +- include_tasks: nova_compute_key_populate.yml + args: + apply: + tags: + - nova-config + - nova-key + when: + - nova_libvirtd_listen_tls == 0 -- import_tasks: nova_compute_key_distribute.yml - tags: - - nova-config - - nova-key +- include_tasks: nova_compute_key_distribute.yml + args: + apply: + tags: + - nova-config + - nova-key + when: + - nova_libvirtd_listen_tls == 0 - name: Run the systemd mount role include_role: diff --git a/templates/libvirtd.conf.j2 b/templates/libvirtd.conf.j2 index 10a8ccb0..b0522e28 100644 --- a/templates/libvirtd.conf.j2 +++ b/templates/libvirtd.conf.j2 @@ -6,7 +6,13 @@ log_level = 1 log_filters="{{ nova_libvirtd_debug_log_filters }}" log_outputs="1:file:/var/log/libvirt/libvirtd.log" {% endif %} +# Flag listening for secure TLS connections on the public TCP/IP port. +# NB, must pass the --listen flag to the libvirtd process for this to +# have any effect. listen_tls = {{ nova_libvirtd_listen_tls }} +# Listen for unencrypted TCP connections on the public TCP/IP port. +# NB, must pass the --listen flag to the libvirtd process for this to +# have any effect. listen_tcp = {{ nova_libvirtd_listen_tcp }} unix_sock_group = "{{ libvirt_group }}" unix_sock_ro_perms = "0777" diff --git a/templates/nova.conf.j2 b/templates/nova.conf.j2 index 7bcef274..bc6fd83b 100644 --- a/templates/nova.conf.j2 +++ b/templates/nova.conf.j2 @@ -239,8 +239,13 @@ images_rbd_pool = {{ nova_libvirt_images_rbd_pool }} images_rbd_ceph_conf = /etc/ceph/ceph.conf {% endif %} {% if nova_virt_type in ['kvm', 'qemu'] %} +{% if nova_libvirtd_listen_tls == 1 %} +live_migration_with_native_tls = true +live_migration_scheme = tls +{% else %} live_migration_uri = "qemu+ssh://nova@%s/system?no_verify=1&keyfile={{ nova_system_home_folder }}/.ssh/id_rsa" live_migration_tunnelled = True +{% endif %} live_migration_inbound_addr = {{ nova_libvirt_live_migration_inbound_addr }} {% endif %} hw_disk_discard = {{ nova_libvirt_hw_disk_discard }} diff --git a/templates/qemu.conf.j2 b/templates/qemu.conf.j2 index 914afec2..e5092a7b 100644 --- a/templates/qemu.conf.j2 +++ b/templates/qemu.conf.j2 @@ -15,6 +15,52 @@ cgroup_device_acl = [ ] {% endif %} +{% if nova_libvirtd_listen_tls == 1 %} +# Use of TLS requires that x509 certificates be issued. The default is +# to keep them in /etc/pki/qemu. This directory must contain +# +# ca-cert.pem - the CA master certificate +# server-cert.pem - the server certificate signed with ca-cert.pem +# server-key.pem - the server private key +# +# and optionally may contain +# +# dh-params.pem - the DH params configuration file +# +# If the directory does not exist, libvirtd will fail to start. If the +# directory doesn't contain the necessary files, QEMU domains will fail +# to start if they are configured to use TLS. +# +# In order to overwrite the default path alter the following. This path +# definition will be used as the default path for other *_tls_x509_cert_dir +# configuration settings if their default path does not exist or is not +# specifically set. +# +default_tls_x509_cert_dir = "{{ nova_qemu_ssl_dir }}" + +# The default TLS configuration only uses certificates for the server +# allowing the client to verify the server's identity and establish +# an encrypted channel. +# +# It is possible to use x509 certificates for authentication too, by +# issuing an x509 certificate to every client who needs to connect. +# +# Enabling this option will reject any client who does not have a +# certificate signed by the CA in /etc/pki/qemu/ca-cert.pem +# +# The default_tls_x509_cert_dir directory must also contain +# +# client-cert.pem - the client certificate signed with the ca-cert.pem +# client-key.pem - the client private key +# +# If this option is supplied it provides the default for the "_verify" option +# of specific TLS users such as vnc, backups, migration, etc. The specific +# users of TLS may override this by setting the specific "_verify" option. +# +# When not supplied the specific TLS users provide their own defaults. +# +default_tls_x509_verify = 1 +{% endif %} {% for key, value in _nova_qemu_conf.items() %} {{ key }} = {{ value }}