From d907a06eff3f5612c90d90d55ef8625e6123f616 Mon Sep 17 00:00:00 2001 From: yuchengde Date: Mon, 12 Jul 2021 13:18:41 +0800 Subject: [PATCH] Separate CA for k8s and etcd Sharing cert between kubernetes-ca and etcd-ca will dramatically increase the complexity of updating etcd-ca cert. Therefore do the separating CA work. Test Plan: 0) General -- PASS check the communication status between apiserver and etcd check the etcd status and configuration on controllers verify applications are upgraded check certs are split check certs are persisted 1) Deployment test on simplex -- PASS (changeset 41) Lock/unlock simplex Simplex spontaneous reboot 2) Deployment test on duplex -- PASS (changeset 41) Switch active controller (changeset 42) Reinstall controller and Swact (changeset 42) 3) Backup and restore starlingx platform on simplex -- PASS (changeset 41) 4) Backup and restore starlingx platform on duplex -- PASS with config-out-of-date on c0 (changeset 41) retest in progress 5) Upgrade starlingx platform on simplex -- PASS (changeset 41) 6) Upgrade starlingx platform on duplex -- PASS 7) Bootstrap replay tests (simplex) - PASS Story: 2008833 Task: 42318 Change-Id: Icc6c922bdc247c3eadf1f54d3f36bbe56f80e17b Signed-off-by: yuchengde Signed-off-by: Michel Thebeau --- .../playbooks/host_vars/bootstrap/default.yml | 2 + .../tasks/apply_bootstrap_manifest.yml | 1 - .../tasks/main.yml | 2 + .../tasks/bringup_kubemaster.yml | 52 +++++++++++---- .../bootstrap/prepare-env/tasks/main.yml | 8 ++- .../bootstrap/validate-config/tasks/main.yml | 16 +++++ .../common/create-etcd-certs/tasks/main.yml | 53 +++++++--------- .../src/playbooks/separate_etcd_ca.yml | 63 +++++++++++++++++++ 8 files changed, 152 insertions(+), 45 deletions(-) create mode 100644 playbookconfig/src/playbooks/separate_etcd_ca.yml diff --git a/playbookconfig/src/playbooks/host_vars/bootstrap/default.yml b/playbookconfig/src/playbooks/host_vars/bootstrap/default.yml index 60150579d..acc8b5236 100644 --- a/playbookconfig/src/playbooks/host_vars/bootstrap/default.yml +++ b/playbookconfig/src/playbooks/host_vars/bootstrap/default.yml @@ -222,6 +222,8 @@ docker_registries: # - ssl_ca_cert # - k8s_root_ca_cert # - k8s_root_ca_key +# - etcd_root_ca_cert +# - etcd_root_ca_key # # ssl_ca_cert: /path/to/ssl_ca_cert_file diff --git a/playbookconfig/src/playbooks/roles/bootstrap/apply-manifest/tasks/apply_bootstrap_manifest.yml b/playbookconfig/src/playbooks/roles/bootstrap/apply-manifest/tasks/apply_bootstrap_manifest.yml index c3230e3f8..f1aa9fa44 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/apply-manifest/tasks/apply_bootstrap_manifest.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/apply-manifest/tasks/apply_bootstrap_manifest.yml @@ -146,7 +146,6 @@ path: "{{ hieradata_workdir }}/static.yaml" line: "{{ item }}" with_items: - - "platform::etcd::params::security_enabled: true" - "platform::etcd::params::bind_address: {{ cluster_floating_address }}" - "platform::etcd::params::bind_address_version: {{ etcd_listen_address_version }}" diff --git a/playbookconfig/src/playbooks/roles/bootstrap/bringup-bootstrap-applications/tasks/main.yml b/playbookconfig/src/playbooks/roles/bootstrap/bringup-bootstrap-applications/tasks/main.yml index d8917f93d..9bc31f267 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/bringup-bootstrap-applications/tasks/main.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/bringup-bootstrap-applications/tasks/main.yml @@ -120,6 +120,8 @@ - "prev_k8s_root_ca_cert: {{ k8s_root_ca_cert }}" - "prev_k8s_root_ca_key: {{ k8s_root_ca_key }}" - "prev_apiserver_oidc: {{ apiserver_oidc | to_yaml }}" + - "prev_etcd_root_ca_cert: {{ etcd_root_ca_cert }}" + - "prev_etcd_root_ca_key: {{ etcd_root_ca_key }}" # Nested dictionaries are picky about having things on the same line - "prev_docker_registries: " - "{{ docker_registries | to_yaml }}" diff --git a/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_kubemaster.yml b/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_kubemaster.yml index c015ffb77..35a3ccd94 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_kubemaster.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/bringup-essential-services/tasks/bringup_kubemaster.yml @@ -81,17 +81,47 @@ - "apiserver-etcd-client.key" when: k8s_pki_files is defined -- name: Copy ca, cert and key generated by etcd to kubeadm_pki_dir - copy: - src: "/etc/etcd/{{ item }}" - dest: "{{ kubeadm_pki_dir }}/{{ item }}" - remote_src: yes - force: yes - with_items: - - "ca.crt" - - "ca.key" - - "apiserver-etcd-client.crt" - - "apiserver-etcd-client.key" +- block: + - name: Copy apiserver cert and key generated by etcd to kubeadm_pki_dir + copy: + src: "/etc/etcd/{{ item }}" + dest: "{{ kubeadm_pki_dir }}/{{ item }}" + remote_src: yes + force: yes + with_items: + - "apiserver-etcd-client.crt" + - "apiserver-etcd-client.key" + + - name: Generate private key for kubernetes-ca + openssl_privatekey: + path: "{{ kubeadm_pki_dir }}/ca.key" + type: RSA + size: 4096 + state: present + force: true + + - name: Generate CSR for kubernetes-ca + openssl_csr: + path: "{{ kubeadm_pki_dir }}/ca.csr" + privatekey_path: "{{ kubeadm_pki_dir }}/ca.key" + common_name: kubernetes + basic_constraints: + - CA:true + - pathlen:1 + basic_constraints_critical: True + key_usage: + - keyCertSign + - digitalSignature + - keyEncipherment + force: true + + - name: Generate self-signed CA certificate for kubernetes-ca + openssl_certificate: + path: "{{ kubeadm_pki_dir }}/ca.crt" + privatekey_path: "{{ kubeadm_pki_dir }}/ca.key" + csr_path: "{{ kubeadm_pki_dir }}/ca.csr" + provider: selfsigned + force: true when: k8s_pki_files is undefined - name: Set kubelet node configuration diff --git a/playbookconfig/src/playbooks/roles/bootstrap/prepare-env/tasks/main.yml b/playbookconfig/src/playbooks/roles/bootstrap/prepare-env/tasks/main.yml index bec14231a..f39285bde 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/prepare-env/tasks/main.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/prepare-env/tasks/main.yml @@ -197,11 +197,13 @@ docker_https_proxy: "{{ docker_https_proxy | default('undef') }}" docker_no_proxy: "{{ docker_no_proxy | default([]) }}" -- name: Set default values for kubernetes certificate parameters if not defined +- name: Define k8s/etcd certificate parameters if not defined set_fact: k8s_root_ca_cert: "{{ k8s_root_ca_cert | default('') }}" k8s_root_ca_key: "{{ k8s_root_ca_key | default('') }}" apiserver_cert_sans: "{{ apiserver_cert_sans | default([]) }}" + etcd_root_ca_cert: "{{ etcd_root_ca_cert | default('') }}" + etcd_root_ca_key: "{{ etcd_root_ca_key | default('') }}" # Give the bootstrap config output file on the host a generic name so the # same file is referenced if the host is bootstrapped locally and remotely @@ -333,7 +335,9 @@ prev_docker_no_proxy != docker_no_proxy | sort)) or (prev_apiserver_cert_sans != apiserver_cert_sans) or (prev_k8s_root_ca_cert != k8s_root_ca_cert) or - (prev_k8s_root_ca_key != k8s_root_ca_key) + (prev_k8s_root_ca_key != k8s_root_ca_key) or + (prev_etcd_root_ca_cert != etcd_root_ca_cert) or + (prev_etcd_root_ca_key != etcd_root_ca_key) - name: Turn on service endpoints reconfiguration flag if management and/or oam network config is changed set_fact: diff --git a/playbookconfig/src/playbooks/roles/bootstrap/validate-config/tasks/main.yml b/playbookconfig/src/playbooks/roles/bootstrap/validate-config/tasks/main.yml index 8cf75d864..e1c91ca35 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/validate-config/tasks/main.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/validate-config/tasks/main.yml @@ -730,6 +730,22 @@ include: validate_address.yml input_address={{ item }} with_items: "{{ apiserver_cert_sans }}" +- name: Verify that either both etcd root ca cert and key are defined or not at all + fail: + msg: "etcd_root_ca_cert and etcd_root_ca_key must be provided as a pair" + when: (etcd_root_ca_cert and not etcd_root_ca_key) or + (not etcd_root_ca_cert and etcd_root_ca_key) + +- name: Check for etcd_root_ca_cert file + fail: + msg: "etcd_root_ca_cert file not found. ({{ etcd_root_ca_cert }})" + when: etcd_root_ca_cert and (not etcd_root_ca_cert is file) + +- name: Check for etcd_root_ca_key file + fail: + msg: "etcd_root_ca_key file not found. ({{ etcd_root_ca_key }})" + when: etcd_root_ca_key and (not etcd_root_ca_key is file) + - name: Verify that either both Kubernetes root ca cert and key are defined or not at all fail: msg: "k8s_root_ca_cert and k8s_root_ca_key must be provided as a pair" diff --git a/playbookconfig/src/playbooks/roles/common/create-etcd-certs/tasks/main.yml b/playbookconfig/src/playbooks/roles/common/create-etcd-certs/tasks/main.yml index fae82ceb3..6eb7b096e 100644 --- a/playbookconfig/src/playbooks/roles/common/create-etcd-certs/tasks/main.yml +++ b/playbookconfig/src/playbooks/roles/common/create-etcd-certs/tasks/main.yml @@ -54,36 +54,27 @@ with_items: - "etcd-client" -- name: Check if CA exists - stat: - path: /etc/kubernetes/pki/ca.crt - register: ca_file +- name: Define values for etcd certificate and key + set_fact: + etcd_root_ca_cert: "{{ etcd_root_ca_cert | default('') }}" + etcd_root_ca_key: "{{ etcd_root_ca_key | default('') }}" -- name: Copy existed CA - copy: - src: "/etc/kubernetes/pki/{{ item }}" - dest: "/etc/etcd/{{ item }}" - remote_src: yes - force: yes - with_items: - - "ca.crt" - - "ca.key" - when: ca_file.stat.exists +- name: Setup dictionary of etcd certificates to install + set_fact: + etcd_ca_files: { ca.crt: "{{etcd_root_ca_cert}}", ca.key: "{{etcd_root_ca_key}}" } + when: + - (etcd_root_ca_cert) + - (etcd_root_ca_key) -- name: copy user specified CA +- name: Copy etcd root ca certificates copy: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - force: yes - with_items: - - { src: "{{ k8s_root_ca_cert }}", dest: "/etc/etcd/ca.crt" } - - { src: "{{ k8s_root_ca_key }}", dest: "/etc/etcd/ca.key" } - when: (k8s_root_ca_cert) + src: "{{ item.value }}" + dest: "/etc/etcd/{{item.key}}" + with_dict: "{{ etcd_ca_files }}" + when: etcd_ca_files is defined - block: - - name: Generate private key for kubernetes-ca - # Reuse this kubernetes-ca for the etcd-ca, - # will copy to /etc/kubernetes/pki later + - name: Generate private key for etcd-ca openssl_privatekey: path: "/etc/etcd/ca.key" type: RSA @@ -91,11 +82,11 @@ state: present force: true - - name: Generate CSR for kubernetes-ca + - name: Generate CSR for etcd-ca openssl_csr: path: "/etc/etcd/ca.csr" privatekey_path: "/etc/etcd/ca.key" - common_name: kubernetes + common_name: etcd basic_constraints: - CA:TRUE - pathlen:1 @@ -103,19 +94,19 @@ key_usage: - keyCertSign - digitalSignature + - keyEncipherment force: true - - name: Generate self-signed CA certificate + - name: Generate self-signed CA certificate for etcd-ca openssl_certificate: path: "/etc/etcd/ca.crt" privatekey_path: "/etc/etcd/ca.key" csr_path: "/etc/etcd/ca.csr" provider: selfsigned force: true + when: etcd_root_ca_cert == '' - when: not ca_file.stat.exists and k8s_root_ca_cert == '' - -- name: Generate certs signed with kubernetes CA certificate" +- name: Generate certs signed with etcd CA certificate openssl_certificate: path: "/etc/etcd/{{ item }}.crt" csr_path: "/etc/etcd/{{ item }}.csr" diff --git a/playbookconfig/src/playbooks/separate_etcd_ca.yml b/playbookconfig/src/playbooks/separate_etcd_ca.yml new file mode 100644 index 000000000..18c8d0841 --- /dev/null +++ b/playbookconfig/src/playbooks/separate_etcd_ca.yml @@ -0,0 +1,63 @@ +--- +# Copyright (c) 2021 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# ROLE DESCRIPTION: +# Create a separate CA cert for etcd, separating from kubernetes +# This file can be removed in the release after STX6.0 +- hosts: all + become: yes + become_user: root + tasks: + - name: Create cert for etcd server and client + import_role: + name: common/create-etcd-certs + + - name: Create etcd cert permdir + file: + path: "{{ config_permdir + '/etcd' }}" + state: directory + mode: 0700 + + - name: Copy etcd certificates to config_permdir + copy: + src: "/etc/etcd/{{ item }}" + dest: "{{ config_permdir + '/etcd' }}/{{ item }}" + remote_src: yes + force: yes + with_items: + - "apiserver-etcd-client.crt" + - "apiserver-etcd-client.key" + - "etcd-server.crt" + - "etcd-server.key" + - "ca.crt" + - "ca.key" + - "etcd-client.crt" + - "etcd-client.key" + + - name: Copy apiserver-etcd-client cert + copy: + src: "/etc/etcd/{{ item }}" + dest: "/etc/kubernetes/pki/{{ item }}" + remote_src: yes + force: yes + with_items: + - "apiserver-etcd-client.crt" + - "apiserver-etcd-client.key" + + - name: Create list of etcd classes to pass to puppet + copy: + dest: "/tmp/etcd.yml" + content: | + classes: + - platform::etcd::upgrade::runtime + + - name: Applying puppet for enabling etcd security + command: > + /usr/local/bin/puppet-manifest-apply.sh + {{ puppet_permdir }}/hieradata/ + {{ ipaddress }} + controller runtime /tmp/etcd.yml + environment: + LC_ALL: "en_US.UTF-8"