Enable TLS for live migrations

Instead of using SSH to live migrate VM's use TLS as this is more
secure and SSH migrations are deprecated.
https://docs.openstack.org/nova/xena/admin/secure-live-migration-with-qemu-native-tls.html
A pre-existing PKI (Public Key Infrastruture) setup is required.

TLS live migrations require that all compute hosts can communcate
with each other on port 16514 and port range 49152 to 49261.

To enable TLS live migrations, both libvirt and QEMU require server
and client certificates, the server certicicates is used to verify
servers and the client cert is used by servers to authenticate
clients. A single cert is created by the pki role, that can be
used by both libvirt and QEMU for both client and server auth.

The client, server and CA certifcates need to installed in a
number of locations on each compute host:
* For Libvirt https://libvirt.org/tlscerts.html
* For QEMU https://github.com/libvirt/libvirt/blob/master/src/qemu/qemu.conf

Depends-On: https://review.opendev.org/c/openstack/ansible-role-pki/+/815007
Depends-On: https://review.opendev.org/c/openstack/ansible-role-pki/+/815849
Depends-On: https://review.opendev.org/c/openstack/ansible-role-pki/+/816857

Change-Id: Iddbe8764bb6d3cd3eaee122b2d5ddc02fa3f7662

defaults/main.yml

@@ -388,7 +388,7 @@ nova_api_threads: "{{ [[(ansible_facts['processor_vcpus']//ansible_facts['proces
 nova_service_in_ldap: "{{ service_ldap_backend_enabled | default(False) }}"
 
 ## libvirtd config options
-nova_libvirtd_listen_tls: 0
+nova_libvirtd_listen_tls: 1
 nova_libvirtd_listen_tcp: 0
 nova_libvirtd_auth_tcp: sasl
 nova_libvirtd_debug_log_filters: "3:remote 4:event 3:json 3:rpc"
@@ -534,3 +534,109 @@ nova_pci_passthrough_whitelist: {}
 #  - '{ "name": "card-alias1", "product_id": "XXXX", "vendor_id": "XXXX" }'
 #  - '{ "name": "card-alias2", "product_id": "XXXY", "vendor_id": "XXXY" }'
 nova_pci_alias: []
+
+# Storage location for SSL certificate authority
+nova_pki_dir: "{{ openstack_pki_dir }}"
+
+# Delegated host for operating the certificate authority
+nova_pki_setup_host: "{{ openstack_pki_setup_host | default('localhost') }}"
+
+# Nova server certificate
+nova_pki_keys_path: "{{ nova_pki_dir ~ '/certs/private/' }}"
+nova_pki_certs_path: "{{ nova_pki_dir ~ '/certs/certs/' }}"
+nova_pki_intermediate_cert_name: "{{ openstack_pki_service_intermediate_cert_name }}"
+nova_pki_intermediate_chain_path: "{{ nova_pki_dir ~ '/roots/' ~ nova_pki_intermediate_cert_name ~ '/certs/' ~ nova_pki_intermediate_cert_name ~ '-chain.crt' }}"
+nova_pki_regen_cert: ''
+# Create client and server cert for compute hosts
+# This certiticate is used during TLS live migrations
+nova_pki_certificates:
+  - name: "nova_{{ ansible_facts['hostname'] }}"
+    provider: ownca
+    cn: "{{ ansible_facts['nodename'] }}"
+    san: "{{ 'DNS:' ~ ansible_facts['hostname'] ~ ',DNS:' ~ ansible_facts['nodename'] ~ ',IP:' ~ (nova_management_address == 'localhost') | ternary('127.0.0.1', nova_management_address) }}"
+    signed_by: "{{ nova_pki_intermediate_cert_name }}"
+    key_usage:
+      - digitalSignature
+      - keyAgreement
+      - keyEncipherment
+    extended_key_usage:
+      - clientAuth
+      - serverAuth
+
+# libvirt default destination files for SSL certificates
+nova_libvirt_ssl_dir: /etc/pki/libvirt
+# QEMU default destination files for SSL certificates
+nova_qemu_ssl_dir: /etc/pki/qemu
+
+# Installation details for SSL certificates for TLS live migration
+nova_pki_install_certificates:
+  # Server certificate used by libvirt for live migrations
+  - src: "{{ nova_user_ssl_cert | default(nova_pki_certs_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '-chain.crt') }}"
+    dest: "{{ nova_libvirt_ssl_dir }}/servercert.pem"
+    owner: "root"
+    group: "root"
+    mode: "0640"
+  # Server certificate key used by libvirt for live migrations
+  - src: "{{ nova_user_ssl_key | default(nova_pki_keys_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '.key.pem') }}"
+    dest: "{{ nova_libvirt_ssl_dir }}/private/serverkey.pem"
+    owner: "root"
+    group: "root"
+    mode: "0640"
+  # Client certificate used by libvirt for live migrations
+  # Defaults to using the server certificate which is signed for both clientAuth and serverAuth
+  - src: "{{ nova_user_ssl_cert | default(nova_pki_certs_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '-chain.crt') }}"
+    dest: "{{ nova_libvirt_ssl_dir }}/clientcert.pem"
+    owner: "root"
+    group: "root"
+    mode: "0640"
+  # Client certificate key used by libvirt for live migrations
+  - src: "{{ nova_user_ssl_key | default(nova_pki_keys_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '.key.pem') }}"
+    dest: "{{ nova_libvirt_ssl_dir }}/private/clientkey.pem"
+    owner: "root"
+    group: "root"
+    mode: "0640"
+  # Server certificate used by QEMU for live migrations
+  - src: "{{ nova_user_ssl_cert | default(nova_pki_certs_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '-chain.crt') }}"
+    dest: "{{ nova_qemu_ssl_dir }}/server-cert.pem"
+    owner: "root"
+    group: "{{ nova_qemu_user }}"
+    mode: "0640"
+  # Server certificate key used by QEMU for live migrations
+  - src: "{{ nova_user_ssl_key | default(nova_pki_keys_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '.key.pem') }}"
+    dest: "{{ nova_qemu_ssl_dir }}/server-key.pem"
+    owner: "root"
+    group: "{{ nova_qemu_user }}"
+    mode: "0640"
+  # Client certificate used by QEMU for live migrations
+  # Defaults to using the server certificate which is signed for both clientAuth and serverAuth
+  - src: "{{ nova_user_ssl_cert | default(nova_pki_certs_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '-chain.crt') }}"
+    dest: "{{ nova_qemu_ssl_dir }}/client-cert.pem"
+    owner: "root"
+    group: "{{ nova_qemu_user }}"
+    mode: "0640"
+  # Client certificate key used by QEMU for live migrations
+  - src: "{{ nova_user_ssl_key | default(nova_pki_keys_path ~ 'nova_' ~ ansible_facts['hostname'] ~ '.key.pem') }}"
+    dest: "{{ nova_qemu_ssl_dir }}/client-key.pem"
+    owner: "root"
+    group: "{{ nova_qemu_user }}"
+    mode: "0640"
+  # Root CA for libvirt
+  # libvirt requires that the CA cert file has any intermediate certificates for the server cert,
+  # so defaults to using the intermediate chain, which contains the intermediate and Root CA
+  - src: "{{ nova_user_ssl_ca_cert | default(nova_pki_intermediate_chain_path) }}"
+    dest: "/etc/pki/CA/cacert.pem"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  # Root CA for qemu
+  - src: "{{ nova_user_ssl_ca_cert | default(nova_pki_intermediate_chain_path) }}"
+    dest: "{{ nova_qemu_ssl_dir }}/ca-cert.pem"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+
+# Define user-provided SSL certificates in:
+# /etc/openstack_deploy/user_variables.yml
+#nova_user_ssl_cert: <path to cert on ansible deployment host>
+#nova_user_ssl_key: <path to cert on ansible deployment host>
+#nova_user_ssl_ca_cert: <path to cert on ansible deployment host>

handlers/main.yml

@@ -20,6 +20,7 @@
     state: "stopped"
   listen:
     - Restart libvirt-bin
+    - "cert installed"
 
 - name: Enable sockets when needed
   service:
@@ -36,6 +37,7 @@
       condition: "{{ nova_libvirtd_listen_tcp | bool }}"
   listen:
     - Restart libvirt-bin
+    - "cert installed"
 
 - name: Start libvirt-bin
   service:
@@ -44,6 +46,7 @@
     state: "started"
   listen:
     - Restart libvirt-bin
+    - "cert installed"
 
 - name: Stop services
   service:

releasenotes/notes/tls-migration-3ed93cc04dab5eee.yaml

@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    Nova now defaults to to using the "QEMU-native TLS" feature
+    for live migrations, rather than the deprecated SSH method.
+    A pre-existing PKI (Public Key Infrastructure) setup is
+    required.
+
+    QEMU-native TLS requires all compute hosts to accept TCP connections on
+    port 16514 and port range 49152 to 49261.
+
+    More information can be found here:
+    https://docs.openstack.org/nova/latest/admin/secure-live-migration-with-qemu-native-tls.html

tasks/main.yml

@@ -34,26 +34,6 @@
   tags:
     - always
 
-- name: Fail if TCP and TLS are both enabled
-  fail:
-    msg: |
-      TCP and TLS connectivity are currently enabled for libvirtd. This
-      combination prevents libvirtd from starting properly since this role
-      does not generate TLS certificates for libvirtd at this time.
-
-      To enable TCP connectivity without TLS, set the following variables:
-
-          nova_libvirtd_listen_tcp: 1
-          nova_libvirtd_listen_tls: 0
-
-      Please note that this configuration does not encrypt communication with
-      libvirtd.
-  when:
-    - nova_libvirtd_listen_tcp == 1
-    - nova_libvirtd_listen_tls == 1
-  tags:
-    - always
-
 - name: Fail if service was deployed using a different installation method
   fail:
     msg: "Switching installation methods for OpenStack services is not supported"
@@ -139,6 +119,22 @@
   tags:
     - nova-config
 
+# Create certs after libvirt groups have been created but before handlers
+- name: Create and install SSL certificates
+  include_role:
+    name: pki
+    tasks_from: main_certs.yml
+  vars:
+    pki_setup_host: "{{ nova_pki_setup_host }}"
+    pki_dir: "{{ nova_pki_dir }}"
+    pki_create_certificates: "{{ nova_user_ssl_cert is not defined and nova_user_ssl_key is not defined }}"
+    pki_regen_certificates: "{{ nova_pki_regen_cert }}"
+    pki_certificates: "{{ nova_pki_certificates }}"
+    pki_install_certificates: "{{ nova_pki_install_certificates }}"
+  when:
+    - nova_libvirtd_listen_tls == 1
+    - "'nova_compute' in group_names"
+
 - import_tasks: nova_post_install.yml
   tags:
     - nova-config

tasks/nova_compute.yml

@@ -21,15 +21,23 @@
   tags:
     - always
 
-- import_tasks: nova_compute_key_populate.yml
-  tags:
-    - nova-config
-    - nova-key
+- include_tasks: nova_compute_key_populate.yml
+  args:
+    apply:
+      tags:
+        - nova-config
+        - nova-key
+  when:
+    - nova_libvirtd_listen_tls == 0
 
-- import_tasks: nova_compute_key_distribute.yml
-  tags:
-    - nova-config
-    - nova-key
+- include_tasks: nova_compute_key_distribute.yml
+  args:
+    apply:
+      tags:
+        - nova-config
+        - nova-key
+  when:
+   - nova_libvirtd_listen_tls == 0
 
 - name: Run the systemd mount role
   include_role:

templates/libvirtd.conf.j2

@@ -6,7 +6,13 @@ log_level = 1
 log_filters="{{ nova_libvirtd_debug_log_filters }}"
 log_outputs="1:file:/var/log/libvirt/libvirtd.log"
 {% endif %}
+# Flag listening for secure TLS connections on the public TCP/IP port.
+# NB, must pass the --listen flag to the libvirtd process for this to
+# have any effect.
 listen_tls = {{ nova_libvirtd_listen_tls }}
+# Listen for unencrypted TCP connections on the public TCP/IP port.
+# NB, must pass the --listen flag to the libvirtd process for this to
+# have any effect.
 listen_tcp = {{ nova_libvirtd_listen_tcp }}
 unix_sock_group = "{{ libvirt_group }}"
 unix_sock_ro_perms = "0777"

templates/nova.conf.j2

@@ -239,8 +239,13 @@ images_rbd_pool = {{ nova_libvirt_images_rbd_pool }}
 images_rbd_ceph_conf = /etc/ceph/ceph.conf
 {% endif %}
 {% if nova_virt_type in ['kvm', 'qemu'] %}
+{% if nova_libvirtd_listen_tls == 1 %}
+live_migration_with_native_tls = true
+live_migration_scheme = tls
+{% else %}
 live_migration_uri = "qemu+ssh://nova@%s/system?no_verify=1&keyfile={{ nova_system_home_folder }}/.ssh/id_rsa"
 live_migration_tunnelled = True
+{% endif %}
 live_migration_inbound_addr = {{ nova_libvirt_live_migration_inbound_addr }}
 {% endif %}
 hw_disk_discard = {{ nova_libvirt_hw_disk_discard }}

templates/qemu.conf.j2

@@ -15,6 +15,52 @@ cgroup_device_acl = [
 ]
 {% endif %}
 
+{% if nova_libvirtd_listen_tls == 1 %}
+# Use of TLS requires that x509 certificates be issued. The default is
+# to keep them in /etc/pki/qemu. This directory must contain
+#
+#  ca-cert.pem - the CA master certificate
+#  server-cert.pem - the server certificate signed with ca-cert.pem
+#  server-key.pem  - the server private key
+#
+# and optionally may contain
+#
+#  dh-params.pem - the DH params configuration file
+#
+# If the directory does not exist, libvirtd will fail to start. If the
+# directory doesn't contain the necessary files, QEMU domains will fail
+# to start if they are configured to use TLS.
+#
+# In order to overwrite the default path alter the following. This path
+# definition will be used as the default path for other *_tls_x509_cert_dir
+# configuration settings if their default path does not exist or is not
+# specifically set.
+#
+default_tls_x509_cert_dir = "{{ nova_qemu_ssl_dir }}"
+
+# The default TLS configuration only uses certificates for the server
+# allowing the client to verify the server's identity and establish
+# an encrypted channel.
+#
+# It is possible to use x509 certificates for authentication too, by
+# issuing an x509 certificate to every client who needs to connect.
+#
+# Enabling this option will reject any client who does not have a
+# certificate signed by the CA in /etc/pki/qemu/ca-cert.pem
+#
+# The default_tls_x509_cert_dir directory must also contain
+#
+#  client-cert.pem - the client certificate signed with the ca-cert.pem
+#  client-key.pem - the client private key
+#
+# If this option is supplied it provides the default for the "_verify" option
+# of specific TLS users such as vnc, backups, migration, etc. The specific
+# users of TLS may override this by setting the specific "_verify" option.
+#
+# When not supplied the specific TLS users provide their own defaults.
+#
+default_tls_x509_verify = 1
+{% endif %}
 
 {% for key, value in _nova_qemu_conf.items() %}
 {{ key }} = {{ value }}